Load lines from the log file with pagination support.
191573
def load(package, prefix, offset=0, limit=1000):
    
    logs = package.all(LogFile, unicode(prefix))
    logs = sorted(logs, key=lambda l: l.name, reverse=True)
    seen = 0
    record = None
    tmp = tempfile.NamedTemporaryFile(suffix='.log')
    for log in logs:
        shutil.copyfileobj(log.fh(), tmp)
        tmp.seek(0)
        for line in reversed(list(tmp)):
            seen += 1
            if seen < offset:
                continue
            if seen > limit:
                tmp.close()
                return
            try:
                d, mo, l, m = line.split(' %s ' % SEP, 4)
                if record is not None:
                    yield record
                record = {'time': d, 'module': mo, 'level': l, 'message': m}
            except ValueError:
                if record is not None:
                    record['message'] += '\n' + line
        tmp.seek(0)
    tmp.close()
    if record is not None:
        yield record
507474
def readline(self, size=-1):
        ""
        while True:
            try:
                record = next(self.reader)
            except StopIteration:
                break

            # Ensure this is a valid record
            if checks.record_is_valid(record):
                if self.use_cache:
                    # Ensures the cache is updated and available
                    self.variants.ensure_cache(record)

                # Calculate the MD5 of the variant itself (not the record)
                md5 = calculate_md5(record)

                # Ensure this variant is not already loaded
                if not self.use_cache or md5 not in self.variants:
                    cleaned = self.process_line(record)
                    cleaned.append(md5)
                    return self.outdel.join(cleaned) + '\n'

        return ''
149824
def do_load(self, filename):
        """"""
        try:
            self.__session.load(filename)
        except IOError as e:
            self.logger.error(e.strerror)
555614
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'logs' in _dict:
            args['logs'] = [Log._from_dict(x) for x in (_dict.get('logs'))]
        else:
            raise ValueError(
                'Required property \'logs\' not present in LogCollection JSON')
        if 'pagination' in _dict:
            args['pagination'] = LogPagination._from_dict(
                _dict.get('pagination'))
        else:
            raise ValueError(
                'Required property \'pagination\' not present in LogCollection JSON'
            )
        return cls(**args)
1026276
def get_lines_from_file(filename, lineno, context_lines):
    

    def get_lines(start, end):
        return [linecache.getline(filename, l).rstrip() for l in range(start, end)]

    lower_bound = max(1, lineno - context_lines)
    upper_bound = lineno + context_lines
    linecache.checkcache(filename)
    pre_context = get_lines(lower_bound, lineno)
    context_line = linecache.getline(filename, lineno).rstrip()
    post_context = get_lines(lineno + 1, upper_bound)
    return lower_bound, pre_context, context_line, post_context
579249
def get_lines(filename: str) -> Iterator[str]:
    """"""
    if filename.endswith('.gz'):
        with gzip.open(filename, 'r') as f:
            for line in f:
                yield line.decode('utf-8')
    else:
        with open(filename, 'r', encoding='utf-8') as f:
            for line in f:
                yield line
1315719
def iter_lines(self, warn_only=False):
        """"""
        remain = ""
        for data in self.iter_content(LINE_CHUNK_SIZE, warn_only=True):
            line_break_found = data[-1] in (b"\n", b"\r")
            lines = data.decode(self.codec).splitlines()
            lines[0] = remain + lines[0]
            if not line_break_found:
                remain = lines.pop()
            for line in lines:
                yield line
        if remain:
            yield remain

        self._state = FINISHED
        if not warn_only:
            self.raise_for_error()
782008
def _load(pathtovector,
              wordlist,
              num_to_load=None,
              truncate_embeddings=None,
              sep=" "):
        """"""
        vectors = []
        addedwords = set()
        words = []

        try:
            wordlist = set(wordlist)
        except ValueError:
            wordlist = set()

        logger.info("Loading {0}".format(pathtovector))

        firstline = open(pathtovector).readline().strip()
        try:
            num, size = firstline.split(sep)
            num, size = int(num), int(size)
            logger.info("Vector space: {} by {}".format(num, size))
            header = True
        except ValueError:
            size = len(firstline.split(sep)) - 1
            logger.info("Vector space: {} dim, # items unknown".format(size))
            word, rest = firstline.split(sep, 1)
            # If the first line is correctly parseable, set header to False.
            header = False

        if truncate_embeddings is None or truncate_embeddings == 0:
            truncate_embeddings = size

        for idx, line in enumerate(open(pathtovector, encoding='utf-8')):

            if header and idx == 0:
                continue

            word, rest = line.rstrip(" \n").split(sep, 1)

            if wordlist and word not in wordlist:
                continue

            if word in addedwords:
                raise ValueError("Duplicate: {} on line {} was in the "
                                 "vector space twice".format(word, idx))

            if len(rest.split(sep)) != size:
                raise ValueError("Incorrect input at index {}, size "
                                 "is {}, expected "
                                 "{}".format(idx+1,
                                             len(rest.split(sep)), size))

            words.append(word)
            addedwords.add(word)
            vectors.append(np.fromstring(rest, sep=sep)[:truncate_embeddings])

            if num_to_load is not None and len(addedwords) >= num_to_load:
                break

        vectors = np.array(vectors).astype(np.float32)

        logger.info("Loading finished")
        if wordlist:
            diff = wordlist - addedwords
            if diff:
                logger.info("Not all items from your wordlist were in your "
                            "vector space: {}.".format(diff))

        return vectors, words
1333541
def lines(query):
	
	filename = support.get_file_name(query)
	if(os.path.isfile(filename)):
		with open(filename) as openfile:
			print len(openfile.readlines())
	else:
		print 'File not found : ' + filename
592257
def get_lines_from_file(filename, lineno, context_lines, loader=None, module_name=None):
    
    source = None
    if loader is not None and hasattr(loader, "get_source"):
        try:
            source = loader.get_source(module_name)
        except ImportError:
            # Traceback (most recent call last):
            #   File "/Users/dcramer/Development/django-sentry/sentry/client/handlers.py", line 31, in emit
            #     get_client().create_from_record(record, request=request)
            #   File "/Users/dcramer/Development/django-sentry/sentry/client/base.py", line 325, in create_from_record
            #     data['__sentry__']['frames'] = varmap(shorten, get_stack_info(stack))
            #   File "/Users/dcramer/Development/django-sentry/sentry/utils/stacks.py", line 112, in get_stack_info
            #     pre_context_lineno, pre_context, context_line, post_context = get_lines_from_file(filename, lineno, 7, loader, module_name)
            #   File "/Users/dcramer/Development/django-sentry/sentry/utils/stacks.py", line 24, in get_lines_from_file
            #     source = loader.get_source(module_name)
            #   File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pkgutil.py", line 287, in get_source
            #     fullname = self._fix_name(fullname)
            #   File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/pkgutil.py", line 262, in _fix_name
            #     "module %s" % (self.fullname, fullname))
            # ImportError: Loader for module cProfile cannot handle module __main__
            source = None
        if source is not None:
            source = source.splitlines()

    if source is None:
        try:
            f = open(filename)
            try:
                source = f.readlines()
            finally:
                f.close()
        except (OSError, IOError):
            pass

    if source is None:
        return []

    encoding = 'ascii'
    for line in source[:2]:
        # File coding may be specified. Match pattern from PEP-263
        # (http://www.python.org/dev/peps/pep-0263/)
        match = _coding_re.search(line)
        if match:
            encoding = match.group(1)
            break
    source = [unicode(sline, encoding, 'replace') for sline in source]

    lower_bound = max(0, lineno - context_lines)
    upper_bound = min(lineno + context_lines + 1, len(source))

    try:
        return [(lineno + 1, source[lineno].strip('\n')) for lineno in xrange(lower_bound, upper_bound)]
    except IndexError:
        # the file may have changed since it was loaded into memory
        return []
687631
def readlines(self, *args, **kwargs):
        """"""
        return list(iter(partial(self.readline, *args, **kwargs), u''))
127923
def readlines_bytes(self):
        """"""

        with open_zipfile_archive(self.path, self.filename) as file:
            for line in file:
                yield line.rstrip(b'\r\n')
333845
def Load(self):
    
    for record in super(EventFileLoader, self).Load():
      yield event_pb2.Event.FromString(record)
971503
def from_file(self, file_name=None):
        """"""

        file_name = self._check_file_name(file_name)

        with open(file_name, 'r') as infile:
            top_level_dict = json.load(infile)

        pages_dict = top_level_dict['info_df']
        pages = pd.DataFrame(pages_dict)
        self.pages = pages
        self.file_name = file_name
        self._prm_packer(top_level_dict['metadata'])
        self.generate_folder_names()
        self.paginate()
1223358
def parse_logs(self, f):
        """"""
        data = {}
        for l in f.splitlines():
            # catch empty lines
            if len(l) == 0:
                continue
            s = l.split("\t")
            data_ = []
            # catch lines with descriptive content: "Of pairs used:"
            for i in s[1:]:
                if len(i) == 0:
                    continue
                try:
                    i.replace('(', '')
                    i.replace(')', '')
                    i.replace(',', '')

                    data_.append(float(i))
                except ValueError:
                    data_.append(i)
            if len(data_) == 0:
                continue
            if s[0].startswith('short range'):
                s[0] = 'short range'
            elif s[0].startswith('same fragment'):
                s[0] = 'same fragment'
            s[0] = s[0].capitalize()
            data[s[0]] = data_
        return data
590573
def parse(logfile, time_format):
  ""

  records = [server.record_from_txt(line, only_elapsed=True,
    time_format=time_format) for line in sys.stdin.readlines()]

  # TODO: make this code better.
  def output(summary):
    width = max([len(p[0]) for p in summary]) + 3
    print '\n'.join([
      "%s%s%s" % (p[0], ' ' * (width - len(p[0])),
        colored(minutes_to_txt(p[1]), 'red')) for p in summary])

  output(server.summarize(records))
76103
def txt_line_iterator(path):
  """"""
  with tf.gfile.Open(path) as f:
    for line in f:
      yield line.strip()