Output the average stats to logger
430783
function printStats() {
  _.each(statsTracker, function(vals, stat) {
    var numStats = vals.length;
    var total = _.reduce(vals, function(memo, num) {
      return memo + num;
    }, 0);
    var avg = parseFloat(total / numStats).toFixed(4) + 'ms';
    var printableTotal = parseFloat(total).toFixed(4) + 'ms';
    logger.log(stat + ': ' + avg + ' (' + printableTotal + ' / ' + numStats + ')');
  });
  statsTracker = {};
}
1030683
def stats(args):
    
    p = OptionParser(stats.__doc__)
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    folder, = args
    statsfiles = iglob(folder, "*.stats")
    after_equal = lambda x: x.split("=")[-1]
    header = "Library Assembled_reads Contigs".split()
    contents = []
    # label=M0096 total=7443 cnts=948 mean=7.851 std=35.96
    for statsfile in statsfiles:
        fp = open(statsfile)
        for row in fp:
            if row.startswith("label="):
                break
        label, total, cnts = row.split()[:3]
        label = after_equal(label)
        reads = int(after_equal(total))
        contigs = int(after_equal(cnts))
        contents.append((label, reads, contigs))

    all_labels, all_reads, all_contigs = zip(*contents)
    contents.append(("SUM", sum(all_reads), sum(all_contigs)))
    contents.append(("AVERAGE (per sample)", \
                    int(np.mean(all_reads)), int(np.mean(all_contigs))))
    contents.append(("MEDIAN (per sample)", \
                    int(np.median(all_reads)), int(np.median(all_contigs))))
    write_csv(header, contents, filename=opts.outfile)
933812
def log_output(f):
    

    @wraps(f)
    def wrapper_fn(*args, **kwargs):
        res = f(*args, **kwargs)
        logging.debug("Logging result %s.", res)
        return res

    return wrapper_fn
420067
def PrintSummary(logfile):
    ''''''
    # Open the log file
    mlog = mavutil.mavlink_connection(filename, notimestamps=args.notimestamps, dialect=args.dialect)

    autonomous_sections = 0 # How many different autonomous sections there are
    autonomous = False # Whether the vehicle is currently autonomous at this point in the logfile
    auto_time = 0.0 # The total time the vehicle was autonomous/guided (seconds)
    start_time = None # The datetime of the first received message (seconds since epoch)
    total_dist = 0.0 # The total ground distance travelled (meters)
    last_gps_msg = None # The first GPS message received
    first_gps_msg = None # The last GPS message received
    true_time = None # Track the first timestamp found that corresponds to a UNIX timestamp

    while True:
        m = mlog.recv_match(condition=args.condition)

        # If there's no match, it means we're done processing the log.
        if m is None:
            break

        # Ignore any failed messages
        if m.get_type() == 'BAD_DATA':
            continue

        # Keep track of the latest timestamp for various calculations
        timestamp = getattr(m, '_timestamp', 0.0)

        # Log the first message time
        if start_time is None:
            start_time = timestamp

        # Log the first timestamp found that is a true timestamp. We first try
        # to get the groundstation timestamp from the log directly. If that fails
        # we try to find a message that outputs a Unix time-since-epoch.
        if true_time is None:
            if not args.notimestamps and timestamp >= 1230768000:
                true_time = timestamp
            elif 'time_unix_usec' in m.__dict__ and m.time_unix_usec >= 1230768000:
                true_time = m.time_unix_usec * 1.0e-6
            elif 'time_usec' in m.__dict__ and m.time_usec >= 1230768000:
                true_time = m.time_usec * 1.0e-6

        # Track the vehicle's speed and status
        if m.get_type() == 'GPS_RAW_INT':

            # Ignore GPS messages without a proper fix
            if m.fix_type < 3 or m.lat == 0 or m.lon == 0:
                continue

            # Log the first GPS location found, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if first_gps_msg is None:
                first_gps_msg = m

            # Track the distance travelled, being sure to skip GPS fixes
            # that are bad (at lat/lon of 0,0)
            if last_gps_msg is None or m.time_usec > last_gps_msg.time_usec or m.time_usec+30e6 < last_gps_msg.time_usec:
                if last_gps_msg is not None:
                    total_dist += distance_two(last_gps_msg, m)

                # Save this GPS message to do simple distance calculations with
                last_gps_msg = m

        elif m.get_type() == 'HEARTBEAT':
            if m.type == mavutil.mavlink.MAV_TYPE_GCS:
                continue
            if (m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED or
                m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == False:
                autonomous = True
                autonomous_sections += 1
                start_auto_time = timestamp
            elif (not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_GUIDED_ENABLED and
                not m.base_mode & mavutil.mavlink.MAV_MODE_FLAG_AUTO_ENABLED) and autonomous == True:
                autonomous = False
                auto_time += timestamp - start_auto_time

    # If there were no messages processed, say so
    if start_time is None:
        print("ERROR: No messages found.")
        return

    # If the vehicle ends in autonomous mode, make sure we log the total time
    if autonomous == True:
        auto_time += timestamp - start_auto_time

    # Compute the total logtime, checking that timestamps are 2009 or newer for validity
    # (MAVLink didn't exist until 2009)
    if true_time:
        start_time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(true_time))
        print("Log started at about {}".format(start_time_str))
    else:
        print("Warning: No absolute timestamp found in datastream. No starting time can be provided for this log.")

    # Print location data
    if last_gps_msg is not None:
        first_gps_position = (first_gps_msg.lat / 1e7, first_gps_msg.lon / 1e7)
        last_gps_position = (last_gps_msg.lat / 1e7, last_gps_msg.lon / 1e7)
        print("Travelled from ({0[0]}, {0[1]}) to ({1[0]}, {1[1]})".format(first_gps_position, last_gps_position))
        print("Total distance : {:0.2f}m".format(total_dist))
    else:
        print("Warning: No GPS data found, can't give position summary.")

    # Print out the rest of the results.
    total_time = timestamp - start_time
    print("Total time (mm:ss): {:3.0f}:{:02.0f}".format(total_time / 60, total_time % 60))
    # The autonomous time should be good, as a steady HEARTBEAT is required for MAVLink operation
    print("Autonomous sections: {}".format(autonomous_sections))
    if autonomous_sections > 0:
        print("Autonomous time (mm:ss): {:3.0f}:{:02.0f}".format(auto_time / 60, auto_time % 60))

    totals.time += total_time
    totals.distance += total_dist
    totals.flights += 1
32002
def average(self):
        ""
        result = DefaultBunch(str)

        #  if possible
        if self.size:
            result.update(
                (key, '' if isinstance(val, basestring) else val / self.size)
                for key, val in self.total.items()
            )

        # Handle time fields
        #for key, fielddef in  engine.FieldDefinition.FIELDS.items():
        #    if key in result and fielddef._matcher is matching.TimeFilter:
        #       result[key] = ''
        #for key, fielddef in  engine.FieldDefinition.FIELDS.items():
        #    if key in result and fielddef._matcher is matching.TimeFilter:
        #        result[key] = engine._fmt_duration(result[key])
        #print self.total
        #print result
        return result
408821
@Scheduled(fixedRate = 60000)
  public void logStatistics() {
    // TODO: do we want to log diff with last log instead?
    if (LOG.isDebugEnabled()) {
      LOG.debug("Cache stats:");
      for (Map.Entry<String, LoadingCache<Query<Entity>, List<Object>>> cacheEntry :
          caches.entrySet()) {
        LOG.debug("{}:{}", cacheEntry.getKey(), cacheEntry.getValue().stats());
      }
    }
  }
31676
def log(self, *msg, sep='', end='\n', flush=False):
        """"""
        if not self.quiet:
            print(*msg, sep=sep, end=end, flush=flush)
848795
def statsAndLoggingAggregator(cls, jobStore, stop, config):
        
        #  Overall timing
        startTime = time.time()
        startClock = getTotalCpuTime()

        def callback(fileHandle):
            statsStr = fileHandle.read()
            if not isinstance(statsStr, str):
                statsStr = statsStr.decode()
            stats = json.loads(statsStr, object_hook=Expando)
            try:
                logs = stats.workers.logsToMaster
            except AttributeError:
                # To be expected if there were no calls to logToMaster()
                pass
            else:
                for message in logs:
                    logger.log(int(message.level),
                               'Got message from job at time %s: %s',
                               time.strftime('%m-%d-%Y %H:%M:%S'), message.text)
            try:
                logs = stats.logs
            except AttributeError:
                pass
            else:
                # we may have multiple jobs per worker
                jobNames = logs.names
                messages = logs.messages
                cls.logWithFormatting(jobNames[0], messages,
                                      message='Received Toil worker log. Disable debug level logging to hide this output')
                cls.writeLogFiles(jobNames, messages, config=config)

        while True:
            # This is a indirect way of getting a message to the thread to exit
            if stop.is_set():
                jobStore.readStatsAndLogging(callback)
                break
            if jobStore.readStatsAndLogging(callback) == 0:
                time.sleep(0.5)  # Avoid cycling too fast

        # Finish the stats file
        text = json.dumps(dict(total_time=str(time.time() - startTime),
                               total_clock=str(getTotalCpuTime() - startClock)), ensure_ascii=True)
        jobStore.writeStatsAndLogging(text)
634769
def analyze_logfile(self, logfile_path):
        self._run_stats['logSource'] = logfile_path
        """"""
        with open(logfile_path) as obj:
            self.analyze_logfile_object(obj)

        self._output_aggregated_report(sys.stdout)

        return 0
802411
def _dump_stats(self):
        '''
        
        '''
        extras = {}
        if 'total' in self.stats_dict:
            self.logger.debug("Compiling total/fail dump stats")
            for key in self.stats_dict['total']:
                final = 'total_{t}'.format(t=key)
                extras[final] = self.stats_dict['total'][key].value()
            for key in self.stats_dict['fail']:
                final = 'fail_{t}'.format(t=key)
                extras[final] = self.stats_dict['fail'][key].value()

        if 'plugins' in self.stats_dict:
            self.logger.debug("Compiling plugin dump stats")
            for name in self.stats_dict['plugins']:
                for key in self.stats_dict['plugins'][name]:
                    final = 'plugin_{n}_{t}'.format(n=name, t=key)
                    extras[final] = self.stats_dict['plugins'][name][key].value()

        if not self.logger.json:
            self.logger.info('Kafka Monitor Stats Dump:\n{0}'.format(
                    json.dumps(extras, indent=4, sort_keys=True)))
        else:
            self.logger.info('Kafka Monitor Stats Dump', extra=extras)
873711
def write_stats(stats, fname, header=None):
    """"""
    # Write stats output to file

    for bg in list(stats.values())[0].keys():
        f = open(fname.format(bg), "w")
        if header:
            f.write(header)
        
        stat_keys = sorted(list(list(stats.values())[0].values())[0].keys())
        f.write("{}\t{}\n".format("Motif", "\t".join(stat_keys)))

        for motif in stats:
            m_stats = stats.get(str(motif), {}).get(bg)
            if m_stats:
                f.write("{}\t{}\n".format(
                    "_".join(motif.split("_")[:-1]),
                    "\t".join([str(m_stats[k]) for k in stat_keys])
                    ))
            else:
                logger.warn("No stats for motif {0}, skipping this motif!".format(motif.id))
            #motifs.remove(motif)
        f.close()

    return
194524
def write_stats(datadfs, outputfile, names=[]):
    
    if outputfile == 'stdout':
        output = sys.stdout
    else:
        output = open(outputfile, 'wt')

    stats = [Stats(df) for df in datadfs]
    features = {
        "Number of reads": "number_of_reads",
        "Total bases": "number_of_bases",
        "Total bases aligned": "number_of_bases_aligned",
        "Median read length": "median_read_length",
        "Mean read length": "mean_read_length",
        "Read length N50": "n50",
        "Average percent identity": "average_identity",
        "Median percent identity": "median_identity",
        "Active channels": "active_channels",
        "Mean read quality": "mean_qual",
        "Median read quality": "median_qual",
    }
    max_len = max([len(k) for k in features.keys()])
    try:
        max_num = max(max([len(str(s.number_of_bases)) for s in stats]),
                      max([len(str(n)) for n in names])) + 6
    except ValueError:
        max_num = max([len(str(s.number_of_bases)) for s in stats]) + 6
    output.write("{:<{}}{}\n".format('General summary:', max_len,
                                     " ".join(['{:>{}}'.format(n, max_num) for n in names])))
    for f in sorted(features.keys()):
        try:
            output.write("{f:{pad}}{v}\n".format(
                f=f + ':',
                pad=max_len,
                v=feature_list(stats, features[f], padding=max_num)))
        except KeyError:
            pass
    if all(["quals" in df for df in datadfs]):
        long_features = {
            "Top 5 longest reads and their mean basecall quality score":
            ["top5_lengths", range(1, 6)],
            "Top 5 highest mean basecall quality scores and their read lengths":
            ["top5_quals", range(1, 6)],
            "Number, percentage and megabases of reads above quality cutoffs":
            ["reads_above_qual", [">Q" + str(q) for q in stats[0].qualgroups]],
        }
        for lf in sorted(long_features.keys()):
            output.write(lf + "\n")
            for i in range(5):
                output.write("{}:\t{}\n".format(
                    long_features[lf][1][i], feature_list(stats, long_features[lf][0], index=i)))
111963
def dump_stats(myStats):
    
    print("\n----%s PYTHON PING Statistics----" % (myStats.thisIP))

    if myStats.pktsSent > 0:
        myStats.fracLoss = (myStats.pktsSent - myStats.pktsRcvd) \
                / myStats.pktsSent

    print(("%d packets transmitted, %d packets received, "
           "%0.1f%% packet loss") % (
        myStats.pktsSent,
        myStats.pktsRcvd,
        100.0 * myStats.fracLoss
    ))

    if myStats.pktsRcvd > 0:
        print("round-trip (ms)  min/avg/max = %d/%0.1f/%d" % (
            myStats.minTime,
            myStats.totTime / myStats.pktsRcvd,
            myStats.maxTime
        ))

    print("")
    return
968552
def log(self, msg, level="info"):
        
        if not self.config.get("log_file"):
            # If level was given as a str then convert to actual level
            level = LOG_LEVELS.get(level, level)
            syslog(level, u"{}".format(msg))
        else:
            # Binary mode so fs encoding setting is not an issue
            with open(self.config["log_file"], "ab") as f:
                log_time = time.strftime("%Y-%m-%d %H:%M:%S")
                # nice formating of data structures using pretty print
                if isinstance(msg, (dict, list, set, tuple)):
                    msg = pformat(msg)
                    # if multiline then start the data output on a fresh line
                    # to aid readability.
                    if "\n" in msg:
                        msg = u"\n" + msg
                out = u"{} {} {}\n".format(log_time, level.upper(), msg)
                try:
                    # Encode unicode strings to bytes
                    f.write(out.encode("utf-8"))
                except (AttributeError, UnicodeDecodeError):
                    # Write any byte strings straight to log
                    f.write(out)
266779
def PrintStats(self):
    """"""
    self.BuildStats()
    self.PrintSummaryTable()
    self.PrintSourceTypeTable()
    self.PrintOSTable()
    self.PrintLabelTable()
755657
def print_stats(self):
        
        stats = self.calculate()
        total_time = '%d:%02d:%02d' % (stats['total_time'] / 3600,
                                       (stats['total_time'] / 3600) / 60,
                                       (stats['total_time'] % 3600) % 60)
        output = """\
Total runtime: {total_time}
    Lyrics found: {found}
    Lyrics not found:{notfound}
    Most useful source:\
{best} ({best_count} lyrics found) ({best_rate:.2f}% success rate)
    Least useful source:\
{worst} ({worst_count} lyrics found) ({worst_rate:.2f}% success rate)
    Fastest website to scrape: {fastest} (Avg: {fastest_time:.2f}s per search)
    Slowest website to scrape: {slowest} (Avg: {slowest_time:.2f}s per search)
    Average time per website: {avg_time:.2f}s

xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
xxx    PER WEBSITE STATS:      xxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
"""
        output = output.format(total_time=total_time,
                               found=stats['found'],
                               notfound=stats['notfound'],
                               best=stats['best'][0].capitalize(),
                               best_count=stats['best'][1],
                               best_rate=stats['best'][2],
                               worst=stats['worst'][0].capitalize(),
                               worst_count=stats['worst'][1],
                               worst_rate=stats['worst'][2],
                               fastest=stats['fastest'][0].capitalize(),
                               fastest_time=stats['fastest'][1],
                               slowest=stats['slowest'][0].capitalize(),
                               slowest_time=stats['slowest'][1],
                               avg_time=self.avg_time())
        for source in sources:
            stat = str(self.source_stats[source.__name__])
            output += f'\n{source.__name__.upper()}\n{stat}\n'

        print(output)
693746
def _log_statistics(self):
        
        rows_per_second_trans = self._count_total / (self._time1 - self._time0)
        rows_per_second_load = self._count_transform / (self._time2 - self._time1)
        rows_per_second_overall = self._count_total / (self._time3 - self._time0)

        self._log('Number of rows processed            : {0:d}'.format(self._count_total))
        self._log('Number of rows transformed          : {0:d}'.format(self._count_transform))
        self._log('Number of rows ignored              : {0:d}'.format(self._count_ignore))
        self._log('Number of rows parked               : {0:d}'.format(self._count_park))
        self._log('Number of errors                    : {0:d}'.format(self._count_error))
        self._log('Number of rows per second processed : {0:d}'.format(int(rows_per_second_trans)))
        self._log('Number of rows per second loaded    : {0:d}'.format(int(rows_per_second_load)))
        self._log('Number of rows per second overall   : {0:d}'.format(int(rows_per_second_overall)))