Split Python source into chunks.

    Chunks are separated by at least two return lines. The break must not
    be followed by a space. Also, long Python strings spanning several lines
    are not splitted.
322265
def _split_python(python):
    
    python = _preprocess(python)
    if not python:
        return []
    lexer = PythonSplitLexer()
    lexer.read(python)
    return lexer.chunks
440836
def readline(self, limit=-1, delim=b'\n'):
        
        self._check_readable()
        chunks = []
        while True:
            chunk = self._buffer.get_chunk(limit, delim)
            if not chunk:
                break
            chunks.append(chunk)
            if chunk.endswith(delim):
                break
            if limit >= 0:
                limit -= len(chunk)
                if limit == 0:
                    break
        if not chunks and not self._buffer.eof and self._buffer.error:
            raise compat.saved_exc(self._buffer.error)
        return b''.join(chunks)
1191313
def multiline_string_lines(source, include_docstrings=False):
    
    line_numbers = set()
    previous_token_type = ''
    try:
        for t in generate_tokens(source):
            token_type = t[0]
            start_row = t[2][0]
            end_row = t[3][0]

            if token_type == tokenize.STRING and start_row != end_row:
                if (
                    include_docstrings or
                    previous_token_type != tokenize.INDENT
                ):
                    # We increment by one since we want the contents of the
                    # string.
                    line_numbers |= set(range(1 + start_row, 1 + end_row))

            previous_token_type = token_type
    except (SyntaxError, tokenize.TokenError):
        pass

    return line_numbers
49283
public static List<String> splitAtLastBlank(String s, int width) {
    List<String> chunks = new ArrayList<>();

    String tmp = s;
    while(tmp.length() > 0) {
      int index = findSplitpoint(tmp, width);
      // store first part
      chunks.add(tmp.substring(0, index));
      // skip whitespace at beginning of line
      while(index < tmp.length() && tmp.charAt(index) == ' ') {
        index += 1;
      }
      // remove a newline
      if(index < tmp.length() && tmp.regionMatches(index, NEWLINE, 0, NEWLINE.length())) {
        index += NEWLINE.length();
      }
      if(index >= tmp.length()) {
        break;
      }
      tmp = tmp.substring(index);
    }

    return chunks;
  }
1327732
def split(text):
    

    # Cleanup text
    text = text.strip()
    text = re.sub('\s+', ' ', text)  # collpse multiple spaces

    space, quote, parts = ' ', '"', []
    part, quoted = '', False

    for char in text:

        # Encoutered beginning double quote
        if char is quote and quoted is False:
            quoted = True
            continue

        # Encountered the ending double quote
        if char is quote and quoted is True:
            quoted = False
            parts.append(part.strip())
            part = ''
            continue

        # Found space in quoted
        if char is space and quoted is True:
            part += char
            continue

        # Found space but not quoted
        if char is space:
            if part:
                parts.append(part)
                part = ''
            continue

        # Found other character
        if char is not space:
            part += char
            continue

    if part:
        parts.append(part.strip())

    return parts
1055869
def split_lines(tokenlist):
    
    line = []

    for item in tokenlist:
        # For (token, text) tuples.
        if len(item) == 2:
            token, string = item
            parts = string.split('\n')

            for part in parts[:-1]:
                if part:
                    line.append((token, part))
                yield line
                line = []

            line.append((token, parts[-1]))
                # Note that parts[-1] can be empty, and that's fine. It happens
                # in the case of [(Token.SetCursorPosition, '')].

        # For (token, text, mouse_handler) tuples.
        #     I know, partly copy/paste, but understandable and more efficient
        #     than many tests.
        else:
            token, string, mouse_handler = item
            parts = string.split('\n')

            for part in parts[:-1]:
                if part:
                    line.append((token, part, mouse_handler))
                yield line
                line = []

            line.append((token, parts[-1], mouse_handler))

    # Always yield the last line, even when this is an empty line. This ensures
    # that when `tokenlist` ends with a newline character, an additional empty
    # line is yielded. (Otherwise, there's no way to differentiate between the
    # cases where `tokenlist` does and doesn't end with a newline.)
    yield line
1168610
def split(text: str) -> List[str]:
    
    return [word for word in SEPARATOR.split(text) if word.strip(' \t')]
149372
def splitlines(self, keepends=False):
        
        # Py2 unicode.splitlines() takes keepends as an optional parameter,
        # not as a keyword argument as in Python 3 str.
        parts = super(newstr, self).splitlines(keepends)
        return [newstr(part) for part in parts]
227193
def parse(source):
    
    if isinstance(source, str):
        return parse_stream(six.StringIO(source))
    else:
        return parse_stream(source)
808015
def sourcelines(self):
        """"""
        # get sourcecode from loader or file
        source = None
        if self.loader is not None:
            try:
                if hasattr(self.loader, 'get_source'):
                    source = self.loader.get_source(self.module)
                elif hasattr(self.loader, 'get_source_by_code'):
                    source = self.loader.get_source_by_code(self.code)
            except Exception:
                # we munch the exception so that we don't cause troubles
                # if the loader is broken.
                pass

        if source is None:
            try:
                f = open(self.filename)
            except IOError:
                return []
            try:
                source = f.read()
            finally:
                f.close()

        # already unicode?  return right away
        if isinstance(source, str):
            return source.splitlines()

        # yes. it should be ascii, but we don't want to reject too many
        # characters in the debugger if something breaks
        charset = 'utf-8'
        if source.startswith(UTF8_COOKIE):
            source = source[3:]
        else:
            for idx, match in enumerate(_line_re.finditer(source)):
                match = _line_re.search(match.group())
                if match is not None:
                    charset = match.group(1)
                    break
                if idx > 1:
                    break

        # on broken cookies we fall back to utf-8 too
        try:
            codecs.lookup(charset)
        except LookupError:
            charset = 'utf-8'

        return source.decode(charset, 'replace').splitlines()
1034939
def split(cls, text):
        
        result = [line.strip('\n') for line in cls.parasep_re.split(text)]
        if result == ['', '']:
            result = ['']
        return result
698787
def parse_string(self, string=''):
        '''
        
        '''
        self.strings = string.splitlines(1)
        self.strings = [self.__parse(line) for line in self.strings]
        return ''.join(self.strings)
452050
def split(string, separator_regexp=None, maxsplit=0):
    
    if not string:
        return []
    if separator_regexp is None:
        separator_regexp = _default_separator()
    if not separator_regexp:
        return string.split()
    return re.split(separator_regexp, string, maxsplit)
1005416
def split(self, url):
        
        parts = url.split('://', 1)
        if len(parts) == 2:
            return parts
        else:
            return (None, url)
503441
def write(self):
        

        # get_chunk may sometimes return bytes, and sometimes return strings
        # because of the nature of the different types of STDIN objects we
        # support
        try:
            chunk = self.get_chunk()
            if chunk is None:
                raise DoneReadingForever

        except DoneReadingForever:
            self.log.debug("done reading")

            if self.tty_in:
                # EOF time
                try:
                    char = termios.tcgetattr(self.stream)[6][termios.VEOF]
                except:
                    char = chr(4).encode()

                # normally, one EOF should be enough to signal to an program
                # that is read()ing, to return 0 and be on your way.  however,
                # some programs are misbehaved, like python3.1 and python3.2.
                # they don't stop reading sometimes after read() returns 0.
                # this can be demonstrated with the following program:
                #
                # import sys
                # sys.stdout.write(sys.stdin.read())
                #
                # then type 'a' followed by ctrl-d 3 times.  in python
                # 2.6,2.7,3.3,3.4,3.5,3.6, it only takes 2 ctrl-d to terminate.
                # however, in python 3.1 and 3.2, it takes all 3.
                #
                # so here we send an extra EOF along, just in case.  i don't
                # believe it can hurt anything
                os.write(self.stream, char)
                os.write(self.stream, char)

            return True

        except NotYetReadyToRead:
            self.log.debug("received no data")
            return False

        # if we're not bytes, make us bytes
        if IS_PY3 and hasattr(chunk, "encode"):
            chunk = chunk.encode(self.encoding)

        for proc_chunk in self.stream_bufferer.process(chunk):
            self.log.debug("got chunk size %d: %r", len(proc_chunk),
                    proc_chunk[:30])

            self.log.debug("writing chunk to process")
            try:
                os.write(self.stream, proc_chunk)
            except OSError:
                self.log.debug("OSError writing stdin chunk")
                return True
949443
def split(self, delimiter=None):
        
        result = []
        strings = self.strings[:]
        current = None

        while strings:
            is_literal, value = strings[0]
            parts = value.split(delimiter, 1)
            if len(parts) > 1:
                value1, value2 = parts
                strings[0] = (is_literal, value2)
                out = EscapedString(value1, is_literal)
                push = True
            else:
                strings = strings[1:]
                out = EscapedString(value, is_literal)
                push = False

            if current is None:
                current = out
            else:
                current = current + out
            if push:
                result.append(current)
                current = None

        if current:
            result.append(current)
        return result
364527
def get_chunks(source, chunk_len):
    
    return (source[i: i + chunk_len] for i in range(0, len(source), chunk_len))