Yield lines of warnings and errors about input bbl lines.

    profiling -- yield also info lines about input bbl file.

    If bbllines is a valid file name, it will be read.
    Else, it should be an iterable of bubble file lines.
193964
def validate(bbllines:iter, *, profiling=False):
    
    if isinstance(bbllines, str):
        if os.path.exists(bbllines):  # filename containing bubble
            bbllines = utils.file_lines(bbllines)
        elif '\n' not in bbllines or '\t' not in bbllines:
            # probably a bad file name: let's rise the proper error
            bbllines = utils.file_lines(bbllines)
        else:  # bubble itself
            bbllines = bbllines.split('\n')
    bubble = tuple(bbllines)
    data = tuple(utils.line_data(line) for line in bubble)
    types = tuple(utils.line_type(line) for line in bubble)
    # launch profiling
    if profiling:
        ltype_counts = Counter(types)
        for ltype, count in ltype_counts.items():
            yield 'INFO {} lines of type {}'.format(count, ltype)
        yield 'INFO {} lines of payload'.format(
            ltype_counts['EDGE'] + ltype_counts['IN'] +
            ltype_counts['NODE'] + ltype_counts['SET'])
    # launch validation
    for errline in (l for l, t in zip(bubble, types) if t == 'ERROR'):
        yield 'ERROR line is not bubble: "{}"'.format(errline)
    tree = BubbleTree.from_bubble_data(data)
    cc, subroots = tree.connected_components()
    # print('cc:', cc)
    # print('subroots:', subroots)
    if profiling:
        yield 'INFO {} top (power)nodes'.format(len(tree.roots))
        yield 'INFO {} connected components'.format(len(cc))
        yield 'INFO {} nodes are defined, {} are used'.format(
            ltype_counts['NODE'], len(tuple(tree.nodes())))
        yield 'INFO {} powernodes are defined, {} are used'.format(
            ltype_counts['SET'], len(tuple(tree.powernodes())))

    yield from inclusions_validation(tree)
    yield from mergeability_validation(tree)
943758
def _yield_estimators(model):
    
    yield from _yield_preproc_steps(model)

    net_prefixes = []
    module_prefixes = []

    if isinstance(model, Pipeline):
        name = model.steps[-1][0]
        net_prefixes.append(name)
        module_prefixes.append(name)
        net = model.steps[-1][1]
    else:
        net = model

    yield '__'.join(net_prefixes), net

    module = net.module
    module_prefixes.append('module')
    yield '__'.join(module_prefixes), module
153751
def gen_lines_from_textfiles(
        files: Iterable[TextIO]) -> Generator[str, None, None]:
    
    for file in files:
        for line in file:
            yield line
179450
def zlines(f = None, sep = "\0", osep = None, size = 8192):     # {{{1
  """"""
  if f is None: f = sys.stdin
  if osep is None: osep = sep
  buf = ""
  while True:
    chars = f.read(size)
    if not chars: break
    buf += chars; lines = buf.split(sep); buf = lines.pop()
    for line in lines: yield line + osep
  if buf: yield buf
1004040
def generate_lines(input_file,
                   start=0,
                   stop=float('inf')):
    """"""
    with gzip.GzipFile(input_file, 'rU') as f:
        for i, line in enumerate(f):
            if i < start:
                continue
            if i >= stop:
                break
            yield line.rstrip()
834975
def _lines(self):
        

        chunkIndices = self._listChunks()
        for chunkIndex in chunkIndices:
            with self._openChunk(chunkIndex) as chunk:
                yield from chunk._lines()
281675
def fasta_iterator(first_line, next_lines)
      current_fasta_header = first_line.chomp
      next_lines.each_slice(batch_size).with_index do |slice, i|
        fasta_mapper = []
        input_set = Set.new

        slice.each do |line|
          line.chomp!
          if fasta? line
            current_fasta_header = line
          else
            fasta_mapper << [current_fasta_header, line]
            input_set << line
          end
        end

        yield(input_set.to_a, i, fasta_mapper)
      end
    end
1048178
def generate_lines(self, infile):
        
        pound = '#'
        
        for line in infile:

            heading = 0

            if line.startswith(pound):
                heading = self.hash_count(line)

            yield dict(line=line, heading=heading)
962028
def inline_bbl(self):
        
        bbl_path = os.path.splitext(self._filepath)[0] + ".bbl"
        try:
            with codecs.open(bbl_path, 'r', encoding='utf-8') as f:
                bbl_text = f.read()
        except IOError:
            print("Cannot open bibliography {0}".format(bbl_path))
        self.text = texutils.inline_bbl(self.text, bbl_text)
97901
def _iter_blocks(self):
        """"""
        lines = []
        for line in self._iter_unfolded_lines():
            if line:
                lines.append(line)
            elif lines:
                self.records_read += 1
                yield lines
                lines = []
        if lines:
            self.records_read += 1
            yield lines
1130463
def readlines(self):
      """"""

      # If the file is not open, there's nothing to return
      if not self._fh:
         raise StopIteration

      at_eof = False
      while True:
         # Clean the buffer sometimes.
         if self._bufoffset > (self._maxreadsize / 2):
            self._buf = self._buf[self._bufoffset:]
            self._bufoffset = 0

         # Fill up the buffer if necessary.
         if len(self._buf) < self._maxreadsize:
            at_eof = not self._read(self._maxreadsize)

         # Look for the next line.
         try:
            next_newline = self._buf.index("\n", self._bufoffset)
            line = self._buf[self._bufoffset:next_newline]
            self._bufoffset = next_newline + 1
            # Save the current file offset for yielding and advance the file offset.
            offset = self._offset
            self._offset += len(line) + 1
            if self._longline:
               # This is the remaining chunk of a long line, we're not going
               # to yield it.
               self._longline = False
            else:
               yield line, offset

         except ValueError:
            # Reached the end of the buffer without finding any newlines.
            if not at_eof:
               # Line is longer than the half the buffer size? - Nope
               logger.warning("Skipping over longline at %s:%d", self._path,
                                                                 self._offset)
               self._bufoffset = len(self._buf) - 1
               self._longline = True
            raise StopIteration
1203359
def lines(self: object, fileids: str, plaintext: bool = True):
        

        for text in self.texts(fileids, plaintext):
            text = re.sub(r'\n\s*\n', '\n', text, re.MULTILINE) # Remove blank lines
            for line in text.split('\n'):
                yield line
1003794
def follow(self, delay=1.0):
        
        # TODO: Handle log file rotation
        self.trailing = True
        unchanged_stats = 0
        
        while not self.should_stop_follow:
            where = self.file.tell()
            line = self.file.readline()
            if line:    
                if self.trailing and line in self.line_terminators:
                    # This is just the line terminator added to the end of the file
                    # before a new line, ignore.
                    self.trailing = False
                    continue

                if line[-1] in self.line_terminators:
                    line = line[:-1]
                    if line[-1:] == '\r\n' and '\r\n' in self.line_terminators:
                        # found crlf
                        line = line[:-1]

                self.trailing = False
                unchanged_stats = 0
                yield line
            else:
                self.trailing = True
                self.seek(where)
                yield no_new_line
                # Try to catch up rotated log file
                unchanged_stats += 1
                if unchanged_stats >= self.MAX_UNCHANGED_STATS and \
                        where != os.stat(self.file.name).st_size:
                    logger.info('Reopen log file because file may has been rotated.')
                    self.reopen_file()

                time.sleep(delay)
1069511
def line_iterator(readable_file, size=None):
    # type: (IO[bytes], Optional[int]) -> Iterator[bytes]
    
    read = readable_file.read
    line = []
    byte = b"1"
    if size is None or size < 0:
        while byte:
            byte = read(1)
            line.append(byte)
            if byte in b"\n":
                yield b"".join(line)
                del line[:]

    else:
        while byte and size:
            byte = read(1)
            size -= len(byte)
            line.append(byte)
            if byte in b"\n" or not size:
                yield b"".join(line)
                del line[:]
24003
def read_examples(input_files, batch_size, shuffle, num_epochs=None):
  """"""
  files = []
  for e in input_files:
    for path in e.split(','):
      files.extend(file_io.get_matching_files(path))
  thread_count = multiprocessing.cpu_count()

  # The minimum number of instances in a queue from which examples are drawn
  # randomly. The larger this number, the more randomness at the expense of
  # higher memory requirements.
  min_after_dequeue = 1000

  # When batching data, the queue's capacity will be larger than the batch_size
  # by some factor. The recommended formula is (num_threads + a small safety
  # margin). For now, we use a single thread for reading, so this can be small.
  queue_size_multiplier = thread_count + 3

  # Convert num_epochs == 0 -> num_epochs is None, if necessary
  num_epochs = num_epochs or None

  # Build a queue of the filenames to be read.
  filename_queue = tf.train.string_input_producer(files, num_epochs, shuffle)

  options = tf.python_io.TFRecordOptions(
      compression_type=tf.python_io.TFRecordCompressionType.GZIP)
  example_id, encoded_example = tf.TFRecordReader(options=options).read_up_to(
      filename_queue, batch_size)

  if shuffle:
    capacity = min_after_dequeue + queue_size_multiplier * batch_size
    return tf.train.shuffle_batch(
        [example_id, encoded_example],
        batch_size,
        capacity,
        min_after_dequeue,
        enqueue_many=True,
        num_threads=thread_count)
  else:
    capacity = queue_size_multiplier * batch_size
    return tf.train.batch(
        [example_id, encoded_example],
        batch_size,
        capacity=capacity,
        enqueue_many=True,
        num_threads=thread_count)
1365149
def read(self):
        """"""

        for line in self.io.read():
            with self.parse_line(line) as j:
                yield j
919399
def _iterate_lines(self):
        """"""
        self._num_lines = 0
        self._restarts = []
        self._rs_state = []

        ln = 0
        for ln, line in enumerate(self.filehandle):
            line = line.decode("utf-8", "replace")
            if (self._has_level is None and
                    line[28:31].strip() in LogEvent.log_levels and
                    line[31:39].strip() in LogEvent.log_components):
                self._has_level = True

            # find version string (fast check to eliminate most lines)
            if "version" in line[:100]:
                logevent = LogEvent(line)
                restart = self._check_for_restart(logevent)
                if restart:
                    self._restarts.append((restart, logevent))

            if "starting :" in line or "starting:" in line:
                # look for hostname, port
                match = re.search('port=(?P<port>\d+).*host=(?P<host>\S+)',
                                  line)
                if match:
                    self._hostname = match.group('host')
                    self._port = match.group('port')

            """ For 3.0 the "[initandlisten] options:" long entry contained the
                "engine" field if WiredTiger was the storage engine. There were
                only two engines, MMAPv1 and WiredTiger
            """
            if "[initandlisten] options:" in line:
                match = re.search('replSet: "(?P<replSet>\S+)"', line)
                if match:
                    self._repl_set = match.group('replSet')

                match = re.search('engine: "(?P<engine>\S+)"', line)
                if match:
                    self._storage_engine = match.group('engine')
                else:
                    self._storage_engine = 'mmapv1'

            """ For 3.2 the "[initandlisten] options:" no longer contains the
                "engine" field So now we have to look for the "[initandlisten]
                wiredtiger_open config:" which was present in 3.0, but would
                now tell us definitively that wiredTiger is being used
            """
            if "[initandlisten] wiredtiger_open config:" in line:
                self._storage_engine = 'wiredTiger'

            if "command admin.$cmd command: { replSetInitiate:" in line:
                match = re.search('{ _id: "(?P<replSet>\S+)", '
                                  'members: (?P<replSetMembers>[^]]+ ])', line)
                if match:
                    self._repl_set = match.group('replSet')
                    self._repl_set_members = match.group('replSetMembers')

            # Replica set config logging in MongoDB 3.0+
            new_config = ("New replica set config in use: ")
            if new_config in line:
                match = re.search('{ _id: "(?P<replSet>\S+)", '
                                  'version: (?P<replSetVersion>\d+), '
                                  '(protocolVersion: (?P<replSetProtocol>\d+), )?'
                                  'members: (?P<replSetMembers>[^]]+ ])', line)
                if match:
                    self._repl_set = match.group('replSet')
                    self._repl_set_members = match.group('replSetMembers')
                    self._repl_set_protocol = match.group('replSetProtocol')
                    self._repl_set_version = match.group('replSetVersion')

            # if ("is now in state" in line and
            #        next(state for state in states if line.endswith(state))):
            if "is now in state" in line:
                tokens = line.split()
                # 2.6
                if tokens[1].endswith(']'):
                    pos = 4
                else:
                    pos = 5
                host = tokens[pos]
                rs_state = tokens[-1]
                state = (host, rs_state, LogEvent(line))
                self._rs_state.append(state)
                continue

            if "[rsMgr] replSet" in line:
                tokens = line.split()
                if self._hostname:
                    host = self._hostname + ':' + self._port
                else:
                    host = os.path.basename(self.name)
                host += ' (self)'
                if tokens[-1] in self.states:
                    rs_state = tokens[-1]
                else:
                    # 2.6
                    if tokens[1].endswith(']'):
                        pos = 2
                    else:
                        pos = 6
                    rs_state = ' '.join(tokens[pos:])

                state = (host, rs_state, LogEvent(line))
                self._rs_state.append(state)
                continue

        self._num_lines = ln + 1

        # reset logfile
        self.filehandle.seek(0)