Yield chunks from a range in a file. No chunk is bigger than maxread.
964895
def _file_iter_range(fp, offset, bytes, maxread=1024*1024):
    ''' '''
    fp.seek(offset)
    while bytes > 0:
        part = fp.read(min(bytes, maxread))
        if not part: break
        bytes -= len(part)
        yield part
906884
def file_page_generator(my_file, max_page_size=2**28):
  
  page_start = "  <page>\n"
  page_end = "  </page>\n"
  chunk_size = max_page_size
  page_start = "  <page>\n"
  page_end = "  </page>\n"
  leftovers = ""
  while True:
    chunk = my_file.read(chunk_size)
    if not chunk:
      break
    chunk = leftovers + chunk
    current_pos = 0
    while True:
      start_pos = chunk.find(page_start, current_pos)
      if start_pos == -1:
        break
      end_pos = chunk.find(page_end, start_pos)
      if end_pos == -1:
        if len(chunk) - start_pos > max_page_size:
          leftovers = ""
        else:
          leftovers = chunk[start_pos:]
        break
      raw_page = chunk[start_pos + len(page_start):end_pos]
      if len(raw_page) < max_page_size:
        ret = parse_page(raw_page)
        if ret:
          yield ret
      current_pos = end_pos + len(page_end)
535555
def _file_iter(input_stream, size, offset=0):
    
    input_stream.seek(offset)
    d = input_stream.read(size)
    while d:
        yield d
        d = input_stream.read(size)
828095
def chunk(seq, n):
    # http://stackoverflow.com/a/312464/190597 (Ned Batchelder)
    
    for i in range(0, len(seq), n):
        yield seq[i:i + n]
1060760
def iter_chunks(l, size):
    
    if isinstance(l, six.integer_types):
        l = six.moves.range(l)

    if is_lazy_iterable(l):
        if size < 1:
            yield list(l)
        else:
            chunk = []
            for elem in l:
                if len(chunk) < size:
                    chunk.append(elem)
                else:
                    yield chunk
                    chunk = [elem]
            else:
                if chunk:
                    yield chunk

    else:
        if size < 1:
            yield l
        else:
            for i in six.moves.range(0, len(l), size):
                yield l[i:i + size]
664405
def get_range_content(ds, start, end):
    ''''''
    if not end or end > ds.info.size:
        end = ds.info.size - 1
    chunksize = 4096

    content_chunks = ds.get_chunked_content(chunksize=chunksize)
    length = 0
    for i in range(int(end/chunksize) + 10):
        chunk_start = chunksize  * i
        chunk_end = chunk_start + chunksize

        # probably shouldn't run out of data, but in case data doesn't
        # match datastream metadata size in fedora...
        try:
            content = next(content_chunks)
        except StopIteration:
            break

        real_chunksize = len(content)

        if chunk_start <= start < chunk_end:
            # start of range is somewhere in the current chunk
            # get the section of requested content at start index
            content = content[start - chunk_start:]

            # range could also *end* in same chunk where it starts
            if chunk_start < end <= chunk_end:

                # trim based on *actual* size of current chunk (before any
                # start trim), since last chunk may not be fullsize
                end_trim = -(chunk_start + real_chunksize - end)
                if end_trim:
                    content = content[:end_trim]
                length += len(content)
                yield content

                # stop - hit the end of the range
                break
            else:
                length += len(content)
                yield content

        elif chunk_start < end <= chunk_end:
            # end of range is in this chunk; trim if necessary, then stop

            # trim based on *actual* size of current chunk (before any
            # start trimming), since last chunk may not be fullsize
            content = content[:-(chunk_start + real_chunksize - end)]

            length += len(content)
            yield content
            # stop - hit the end of the range
            break

        elif chunk_start > start  and chunk_end < end:
            # chunk is somewhere in the range of start - end
            length += len(content)
            yield content

    logger.debug('total content length returned is %d', length)
1089335
def parse_range_header(header, maxlen=0):
    ''' '''
    if not header or header[:6] != 'bytes=': return
    ranges = [r.split('-', 1) for r in header[6:].split(',') if '-' in r]
    for start, end in ranges:
        try:
            if not start:  # bytes=-100    -> last 100 bytes
                start, end = max(0, maxlen-int(end)), maxlen
            elif not end:  # bytes=100-    -> all but the first 99 bytes
                start, end = int(start), maxlen
            else:          # bytes=100-200 -> bytes 100-200 (inclusive)
                start, end = int(start), min(int(end)+1, maxlen)
            if 0 <= start < end <= maxlen:
                yield start, end
        except ValueError:
            pass
24047
def read_in_chunks(file_obj, chunk_size):
    """"""
    offset = 0
    while True:
        data = file_obj.read(chunk_size)
        if not data:
            break
        yield data, offset
        offset += len(data)
1069511
def line_iterator(readable_file, size=None):
    # type: (IO[bytes], Optional[int]) -> Iterator[bytes]
    
    read = readable_file.read
    line = []
    byte = b"1"
    if size is None or size < 0:
        while byte:
            byte = read(1)
            line.append(byte)
            if byte in b"\n":
                yield b"".join(line)
                del line[:]

    else:
        while byte and size:
            byte = read(1)
            size -= len(byte)
            line.append(byte)
            if byte in b"\n" or not size:
                yield b"".join(line)
                del line[:]
39203
def chunks(iterable, size = 8):
	
	for i in range(0, len(iterable), size):
		yield l[i:i + n]
375098
def _chunks(l, ncols):
    """"""
    assert isinstance(ncols, int), "ncols must be an integer"
    for i in range(0, len(l), ncols):
        yield l[i: i+ncols]
941653
def generate_chunks(string, num_chars):
    """"""
    for start in range(0, len(string), num_chars):
        yield string[start:start+num_chars]
642710
def read_chunks(filepath, chunk_size):
    """"""
    try:
        with open(filepath, 'rb') as f:
            while True:
                chunk = f.read(chunk_size)
                if chunk:
                    yield chunk
                else:
                    break  # EOF
    except OSError as e:
        raise error.ReadError(e.errno, filepath)
1182675
def read_until_yieldable(self):
        """"""
        while not self.yieldable():
            read_content, read_position = _get_next_chunk(self.fp, self.read_position, self.chunk_size)
            self.add_to_buffer(read_content, read_position)
70635
def reader(stream, fieldnames=None):
    """"""
    if not fieldnames:
        fieldnames = load_line(stream.readline())
    for line in stream:
        values = load_line(line)
        item = Item()
        item.__dict__ = dict(zip(fieldnames, values))
        yield item
1075333
def BackwardsReader(file, BLKSIZE = 4096):
    """"""

    buf = ""

    file.seek(0, 2)
    lastchar = file.read(1)
    trailing_newline = (lastchar == "\n")

    while 1:
        newline_pos = buf.rfind("\n")
        pos = file.tell()
        if newline_pos != -1:
            # Found a newline
            line = buf[newline_pos+1:]
            buf = buf[:newline_pos]
            if pos or newline_pos or trailing_newline:
                line += "\n"
            yield line
        elif pos:
            # Need to fill buffer
            toread = min(BLKSIZE, pos)
            file.seek(pos-toread, 0)
            buf = file.read(toread) + buf
            file.seek(pos-toread, 0)
            if pos == toread:
                buf = "\n" + buf
        else:
            # Start-of-file
            return
1104534
def chunk(iterable, size):
    
    # TODO: only used in gui.mainwindow(deprecated)
    iterator = iter(iterable)
    while size:
        result = []
        try:
            for i in range(size):
                elem = next(iterator)
                result.append(elem)
            yield tuple(result)
        except StopIteration:
            if result:
                yield tuple(result)
            return