Morph log data by timestamp range (single log)
197915
def filter_data(self, data, value=None, utc=False, args=None):
        """"""
        if args:
            if not args.range:
                return data
        if not value:
            value = args.range
            utc = args.utc
        ourlimits = value.split('-')

        newdata = {}
        if 'parser' in data.keys():
            newdata['parser'] = data['parser']
            newdata['source_path'] = data['source_path']
            newdata['source_file'] = data['source_file']
            newdata['source_file_mtime'] = data['source_file_mtime']
            newdata['source_file_year'] = data['source_file_year']
        newdata['entries'] = []

        firstdate = int(ourlimits[0].ljust(14, '0'))
        lastdate = int(ourlimits[1].ljust(14, '0'))
        for entry in data['entries']:
            if utc:
                if 'numeric_date_stamp_utc' in entry:
                    if 'numeric_date_stamp_utc' in entry:
                        if '.' in entry['numeric_date_stamp_utc']:
                            dstamp = int(
                                    entry['numeric_date_stamp_utc'].split(
                                        '.')[0])
                        else:
                            dstamp = int(entry['numeric_date_stamp_utc'])
                        if dstamp >= firstdate:
                            if dstamp <= lastdate:
                                newdata['entries'].append(entry)
            else:
                if 'numeric_date_stamp' in entry:
                    if '.' in entry['numeric_date_stamp']:
                        dstamp = int(
                                entry['numeric_date_stamp'].split('.')[0])
                    else:
                        dstamp = int(entry['numeric_date_stamp'])
                    if dstamp >= firstdate:
                        if dstamp <= lastdate:
                            newdata['entries'].append(entry)

        return newdata
377808
def bulk_log_data(registers = register_list)
      registers.map do |reg|
        read_scaled_range(reg.start, reg.count)
      end.flatten
    end
508797
def _tail_profile(self, db, interval):
        """"""
        latest_doc = None
        while latest_doc is None:
            time.sleep(interval)
            latest_doc = db['system.profile'].find_one()

        current_time = latest_doc['ts']

        while True:
            time.sleep(interval)
            cursor = db['system.profile'].find({'ts': {'$gte': current_time}}).sort('ts', pymongo.ASCENDING)
            for doc in cursor:
                current_time = doc['ts']
                yield doc
11011
def _stab_log_data(self, timestamp, data, logconf):
        """"""
        print('[%d][%s]: %s' % (timestamp, logconf.name, data))
1332203
def _parse_logline_timestamp(t):
    
    date, time = t.split(' ')
    month, day = date.split('-')
    h, m, s = time.split(':')
    s, ms = s.split('.')
    return (month, day, h, m, s, ms)
102095
def interpolate_timestamp(capture_times):
    '''
    
    '''
    timestamps = []
    num_file = len(capture_times)

    time_dict = OrderedDict()

    if num_file < 2:
        return capture_times

    # trace identical timestamps (always assume capture_times is sorted)
    time_dict = OrderedDict()
    for i, t in enumerate(capture_times):
        if t not in time_dict:
            time_dict[t] = {
                "count": 0,
                "pointer": 0
            }

            interval = 0
            if i != 0:
                interval = (t - capture_times[i - 1]).total_seconds()
                time_dict[capture_times[i - 1]]["interval"] = interval

        time_dict[t]["count"] += 1

    if len(time_dict) >= 2:
        # set time interval as the last available time interval
        time_dict[time_dict.keys()[-1]
                  ]["interval"] = time_dict[time_dict.keys()[-2]]["interval"]
    else:
        # set time interval assuming capture interval is 1 second
        time_dict[time_dict.keys()[0]]["interval"] = time_dict[time_dict.keys()[
            0]]["count"] * 1.

    # interpolate timestamps
    for t in capture_times:
        d = time_dict[t]
        s = datetime.timedelta(
            seconds=d["pointer"] * d["interval"] / float(d["count"]))
        updated_time = t + s
        time_dict[t]["pointer"] += 1
        timestamps.append(updated_time)

    return timestamps
622804
def _calculate_bounds(self):
        """"""
        if self._bounds_calculated:
            # Assume no need to recalc bounds for lifetime of a Logfile object
            return

        if self.from_stdin:
            return False

        # we should be able to find a valid log line within max_start_lines
        max_start_lines = 10
        lines_checked = 0

        # get start datetime
        for line in self.filehandle:
            logevent = LogEvent(line)
            lines_checked += 1
            if logevent.datetime:
                self._start = logevent.datetime
                self._timezone = logevent.datetime.tzinfo
                self._datetime_format = logevent.datetime_format
                self._datetime_nextpos = logevent._datetime_nextpos
                break
            if lines_checked > max_start_lines:
                break

        # sanity check before attempting to find end date
        if (self._start is None):
            raise SystemExit("Error: <%s> does not appear to be a supported "
                             "MongoDB log file format" % self.filehandle.name)

        # get end datetime (lines are at most 10k,
        # go back 30k at most to make sure we catch one)
        self.filehandle.seek(0, 2)
        self._filesize = self.filehandle.tell()
        self.filehandle.seek(-min(self._filesize, 30000), 2)

        for line in reversed(self.filehandle.readlines()):
            logevent = LogEvent(line)
            if logevent.datetime:
                self._end = logevent.datetime
                break

        # if there was a roll-over, subtract 1 year from start time
        if self._end < self._start:
            self._start = self._start.replace(year=self._start.year - 1)
            self._year_rollover = self._end
        else:
            self._year_rollover = False

        # reset logfile
        self.filehandle.seek(0)
        self._bounds_calculated = True

        return True
1106370
def logsumexp(arr,axis=0):
    """"""
    minarr= numpy.amax(arr,axis=axis)
    if axis == 1:
        minarr= numpy.reshape(minarr,(arr.shape[0],1))
    if axis == 0:
        minminarr= numpy.tile(minarr,(arr.shape[0],1))
    elif axis == 1:
        minminarr= numpy.tile(minarr,(1,arr.shape[1]))
    elif axis == None:
        minminarr= numpy.tile(minarr,arr.shape)
    else:
        raise NotImplementedError("'galpy.util.logsumexp' not implemented for axis > 2")
    if axis == 1:
        minarr= numpy.reshape(minarr,(arr.shape[0]))
    return minarr+numpy.log(numpy.sum(numpy.exp(arr-minminarr),axis=axis))
1360778
def _GetTimelineStatEntriesLegacy(client_id, file_path, with_history=True):
  """"""

  folder_urn = aff4.ROOT_URN.Add(str(client_id)).Add(file_path)

  child_urns = []
  for _, children in aff4.FACTORY.RecursiveMultiListChildren([folder_urn]):
    child_urns.extend(children)

  if with_history:
    timestamp = aff4.ALL_TIMES
  else:
    timestamp = aff4.NEWEST_TIME

  for fd in aff4.FACTORY.MultiOpen(child_urns, age=timestamp):
    file_path = "/".join(str(fd.urn).split("/")[2:])

    if not with_history:
      yield file_path, fd.Get(fd.Schema.STAT), fd.Get(fd.Schema.HASH)
      continue

    result = {}

    stats = fd.GetValuesForAttribute(fd.Schema.STAT)
    for s in stats:
      result[s.age] = [s, None]

    hashes = fd.GetValuesForAttribute(fd.Schema.HASH)
    for h in hashes:
      prev = result.setdefault(h.age, [None, None])
      prev[1] = h

    for ts in sorted(result):
      v = result[ts]
      yield file_path, v[0], v[1]
846605
def dump_data(logdir):
  """"""
  # Create a tfevents file in the logdir so it is detected as a run.
  write_empty_event_file(logdir)

  plugin_logdir = plugin_asset_util.PluginDirectory(
      logdir, profile_plugin.ProfilePlugin.plugin_name)
  _maybe_create_directory(plugin_logdir)

  for run in profile_demo_data.RUNS:
    run_dir = os.path.join(plugin_logdir, run)
    _maybe_create_directory(run_dir)
    if run in profile_demo_data.TRACES:
      with open(os.path.join(run_dir, 'trace'), 'w') as f:
        proto = trace_events_pb2.Trace()
        text_format.Merge(profile_demo_data.TRACES[run], proto)
        f.write(proto.SerializeToString())

    if run not in profile_demo_data.TRACE_ONLY:
      shutil.copyfile('tensorboard/plugins/profile/profile_demo.op_profile.json',
                      os.path.join(run_dir, 'op_profile.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.memory_viewer.json',
          os.path.join(run_dir, 'memory_viewer.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.pod_viewer.json',
          os.path.join(run_dir, 'pod_viewer.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.google_chart_demo.json',
          os.path.join(run_dir, 'google_chart_demo.json'))

  # Unsupported tool data should not be displayed.
  run_dir = os.path.join(plugin_logdir, 'empty')
  _maybe_create_directory(run_dir)
  with open(os.path.join(run_dir, 'unsupported'), 'w') as f:
    f.write('unsupported data')
173113
function morph (value, input) {
  const result = input || readable()
  const cb = map[type(value)] || end
  cb(result, value)
  return new Proxy(result, {
    get(target, key, receiver) {
      if (key !== 'pipe') return target[key]
      return function (dest) {
        pump(result, dest)
        return dest
      }
    }
  })
}
44916
def _dump_query_timestamps(self, current_time: float):
        """"""
        windows = [10, 11, 15, 20, 30, 60]
        print("GraphQL requests:", file=sys.stderr)
        for query_hash, times in self._graphql_query_timestamps.items():
            print("  {}".format(query_hash), file=sys.stderr)
            for window in windows:
                reqs_in_sliding_window = sum(t > current_time - window * 60 for t in times)
                print("    last {} minutes: {} requests".format(window, reqs_in_sliding_window), file=sys.stderr)
994949
function getChangelog(log) {
      var data = {
        date: moment().format('YYYY-MM-DD'),
        features: getChanges(log, options.featureRegex),
        fixes: getChanges(log, options.fixRegex)
      };

      return template(data);
    }
22187
def analyze_log_file(logfile, pattern, reverse_paths=True, progress=True):
    ""
    if progress:
        lines = count_lines_in(logfile)
        pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=lines+1).start()
        counter = 0
    
    data = {}
    
    compiled_pattern = compile(pattern)
    for line in fileinput.input([logfile]):
        
        if progress:
            counter = counter + 1
        
        parsed = compiled_pattern.findall(line)[0]
        date = parsed[0]
        method = parsed[1]
        path = parsed[2]
        status = parsed[3]
        time = parsed[4]
        sql = parsed[5]
        sqltime = parsed[6]

        try:
            ignore = False
            for ignored_path in IGNORE_PATHS:
                compiled_path = compile(ignored_path)
                if compiled_path.match(path):
                    ignore = True
            if not ignore:
                if reverse_paths:
                    view = view_name_from(path)
                else:
                    view = path
                key = "%s-%s-%s" % (view, status, method)
                try:
                    data[key]['count'] = data[key]['count'] + 1
                    data[key]['times'].append(float(time))
                    data[key]['sql'].append(int(sql))
                    data[key]['sqltime'].append(float(sqltime))
                except KeyError:
                    data[key] = {
                        'count': 1,
                        'status': status,
                        'view': view,
                        'method': method,
                        'times': [float(time)],
                        'sql': [int(sql)],
                        'sqltime': [float(sqltime)],
                    }
        except Resolver404:
            pass
        
        if progress:
            pbar.update(counter)
    
    if progress:
        pbar.finish()
    
    return data
590573
def parse(logfile, time_format):
  ""

  records = [server.record_from_txt(line, only_elapsed=True,
    time_format=time_format) for line in sys.stdin.readlines()]

  # TODO: make this code better.
  def output(summary):
    width = max([len(p[0]) for p in summary]) + 3
    print '\n'.join([
      "%s%s%s" % (p[0], ' ' * (width - len(p[0])),
        colored(minutes_to_txt(p[1]), 'red')) for p in summary])

  output(server.summarize(records))
295945
def process_file(filename, timeshift):
    ''''''
    print("Processing %s" % filename)
    mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, zero_time_base=args.zero_time_base, dialect=args.dialect)
    vars = {}

    while True:
        msg = mlog.recv_match(args.condition)
        if msg is None: break
        try:
            tdays = matplotlib.dates.date2num(datetime.datetime.fromtimestamp(msg._timestamp+timeshift))
        except ValueError:
            # this can happen if the log is corrupt
            # ValueError: year is out of range
            break
        add_data(tdays, msg, mlog.messages, mlog.flightmode)
591651
def to_timestamp(self, data):
        
        result = pd.Series(index=data.index)
        _slice = ~data[self.col_name].isnull()

        result[_slice] = data[_slice][self.col_name].astype('int64')
        return result