Counts hunt output plugin log entries.
969554
def CountHuntOutputPluginLogEntries(self,
                                      hunt_id,
                                      output_plugin_id,
                                      with_type=None):
    """"""

    return len(
        self.ReadHuntOutputPluginLogEntries(
            hunt_id, output_plugin_id, 0, sys.maxsize, with_type=with_type))
975397
def log_every_x_times(logger, counter, x, msg, *args, **kwargs):
    '''
    
    '''
    if counter==1 or counter % x == 0:
        #msg = msg + (' (counter %i)' % counter)
        logdebug(logger, msg, *args, **kwargs)
55700
def WriteHuntOutputPluginsStates(self, hunt_id, states, cursor=None):
    """"""

    columns = ", ".join(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS)
    placeholders = mysql_utils.Placeholders(
        2 + len(_HUNT_OUTPUT_PLUGINS_STATES_COLUMNS))
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    for index, state in enumerate(states):
      query = ("INSERT INTO hunt_output_plugins_states "
               "(hunt_id, plugin_id, {columns}) "
               "VALUES {placeholders}".format(
                   columns=columns, placeholders=placeholders))
      args = [hunt_id_int, index, state.plugin_descriptor.plugin_name]

      if state.plugin_descriptor.plugin_args is None:
        args.append(None)
      else:
        args.append(state.plugin_descriptor.plugin_args.SerializeToString())

      args.append(state.plugin_state.SerializeToString())

      try:
        cursor.execute(query, args)
      except MySQLdb.IntegrityError as e:
        raise db.UnknownHuntError(hunt_id=hunt_id, cause=e)
292256
def CallLoggedAndAccounted(f):
  """"""

  @functools.wraps(f)
  def Decorator(*args, **kwargs):
    try:
      start_time = time.time()
      result = f(*args, **kwargs)
      latency = time.time() - start_time

      stats_collector_instance.Get().RecordEvent(
          "db_request_latency", latency, fields=[f.__name__])
      logging.debug("DB request %s SUCCESS (%.3fs)", f.__name__, latency)

      return result
    except db.Error as e:
      stats_collector_instance.Get().IncrementCounter(
          "db_request_errors", fields=[f.__name__, "grr"])
      logging.debug("DB request %s GRR ERROR: %s", f.__name__,
                    utils.SmartUnicode(e))
      raise
    except Exception as e:
      stats_collector_instance.Get().IncrementCounter(
          "db_request_errors", fields=[f.__name__, "db"])
      logging.debug("DB request %s INTERNAL DB ERROR : %s", f.__name__,
                    utils.SmartUnicode(e))
      raise

  return Decorator
1120952
def ReadHuntResults(self,
                      hunt_id,
                      offset,
                      count,
                      with_tag=None,
                      with_type=None,
                      with_substring=None,
                      with_timestamp=None,
                      cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = ("SELECT client_id, flow_id, hunt_id, payload, type, "
             "UNIX_TIMESTAMP(timestamp), tag "
             "FROM flow_results "
             "FORCE INDEX(flow_results_hunt_id_flow_id_timestamp) "
             "WHERE hunt_id = %s ")

    args = [hunt_id_int]

    if with_tag:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type:
      query += "AND type = %s "
      args.append(with_type)

    if with_substring:
      query += "AND payload LIKE %s "
      args.append("%" + db_utils.EscapeWildcards(with_substring) + "%")

    if with_timestamp:
      query += "AND timestamp = FROM_UNIXTIME(%s) "
      args.append(mysql_utils.RDFDatetimeToTimestamp(with_timestamp))

    query += "ORDER BY timestamp ASC LIMIT %s OFFSET %s"
    args.append(count)
    args.append(offset)

    cursor.execute(query, args)

    ret = []
    for (
        client_id_int,
        flow_id_int,
        hunt_id_int,
        serialized_payload,
        payload_type,
        timestamp,
        tag,
    ) in cursor.fetchall():
      if payload_type in rdfvalue.RDFValue.classes:
        payload = rdfvalue.RDFValue.classes[payload_type]()
        payload.ParseFromString(serialized_payload)
      else:
        payload = rdf_objects.SerializedValueOfUnrecognizedType(
            type_name=payload_type, value=serialized_payload)

      result = rdf_flow_objects.FlowResult(
          client_id=db_utils.IntToClientID(client_id_int),
          flow_id=db_utils.IntToFlowID(flow_id_int),
          hunt_id=hunt_id,
          payload=payload,
          timestamp=mysql_utils.TimestampToRDFDatetime(timestamp))
      if tag is not None:
        result.tag = tag

      ret.append(result)

    return ret
858736
function tally(pathname, offenses) {
    var count = offenses.length,
        plural = count > 1 ? 'errors' : 'error';

    function perOffense(o) {
        if (!errors[pathname]) {
            errors[pathname] = [];
        }

        if (o) {
            errors[pathname].push({
                line: o.line,
                col: o.character,
                msg: o.reason,
                evidence: o.evidence || ''
            });
        }
    }

    log.error([count, plural, 'in', pathname].join(' '));
    offenses.forEach(perOffense);
}
782228
def ReadFlowOutputPluginLogEntries(self,
                                     client_id,
                                     flow_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """"""
    entries = sorted(
        self.flow_output_plugin_log_entries.get((client_id, flow_id), []),
        key=lambda e: e.timestamp)

    entries = [e for e in entries if e.output_plugin_id == output_plugin_id]

    if with_type is not None:
      entries = [e for e in entries if e.log_entry_type == with_type]

    return entries[offset:offset + count]
957997
def log_summary(self):
        """"""
        participants = Participant.query.with_entities(Participant.status).all()
        counts = Counter([p.status for p in participants])
        sorted_counts = sorted(counts.items(), key=itemgetter(0))
        self.log("Status summary: {}".format(str(sorted_counts)))
        return sorted_counts
613135
def log_stats(self):
        """"""
        if not self.stats.get('counts'):
            if self.consumers:
                LOGGER.info('Did not receive any stats data from children')
            return

        if self.poll_data['processes']:
            LOGGER.warning('%i process(es) did not respond with stats: %r',
                           len(self.poll_data['processes']),
                           self.poll_data['processes'])

        if self.stats['counts']['processes'] > 1:
            LOGGER.info('%i consumers processed %i messages with %i errors',
                        self.stats['counts']['processes'],
                        self.stats['counts']['processed'],
                        self.stats['counts']['failed'])

        for key in self.stats['consumers'].keys():
            LOGGER.info('%i %s %s processed %i messages with %i errors',
                        self.stats['consumers'][key]['processes'], key,
                        self.consumer_keyword(self.stats['consumers'][key]),
                        self.stats['consumers'][key]['processed'],
                        self.stats['consumers'][key]['failed'])
502843
def constructTotalCounts(self, logger):
        '''
        
        '''
        self.totalSize = self.bwt.shape[0]
        
        abtFN = self.dirName+'/totalCounts.p'
        if os.path.exists(abtFN):
            fp = open(abtFN, 'r')
            self.totalCounts = pickle.load(fp)
            fp.close()
        else:
            chunkSize = 2**20
            if logger != None:
                logger.info('First time calculation of \'%s\'' % abtFN)
            
            #figure out the counts using the standard counting techniques, one chunk at a time
            self.totalCounts = [0]*self.vcLen
            i = 0
            while i*chunkSize < self.bwt.shape[0]:
                self.totalCounts = np.add(self.totalCounts, np.bincount(self.bwt[i*chunkSize:(i+1)*chunkSize], minlength=self.vcLen))
                i += 1
            
            #save the total count to '<DIR>/totalCounts.p'
            fp = open(abtFN, 'w+')
            pickle.dump(self.totalCounts, fp)
            fp.close()
1052641
def searchlog(self, argv):
        """"""
        opts = cmdline(argv, FLAGS_SEARCHLOG)
        self.foreach(opts.args, lambda job: 
            output(job.searchlog(**opts.kwargs)))
1016734
def CountHuntResults(self,
                       hunt_id,
                       with_tag=None,
                       with_type=None,
                       cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = "SELECT COUNT(*) FROM flow_results WHERE hunt_id = %s "

    args = [hunt_id_int]

    if with_tag is not None:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type is not None:
      query += "AND type = %s "
      args.append(with_type)

    cursor.execute(query, args)
    return cursor.fetchone()[0]
25553
def count(args):
    
    p = OptionParser(count.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(p.print_help())

    frgfile, = args
    fp = open(frgfile)

    counts = defaultdict(int)
    for rec in iter_records(fp):
        counts[rec.type] += 1

    for type, cnt in sorted(counts.items()):
        print('{0}: {1}'.format(type, cnt), file=sys.stderr)
373254
function logReports (url, reports, verbose) {
    var passes = '';
    var failures = '';

    grunt.log.writeln(chalk.underline(chalk.cyan('\nReport for ' + url + '\n')));
    reports.audit.forEach(function (el) {
        if (el.result === 'PASS') {
            passes += logSymbols.success + ' ' + el.heading + '\n';
        }

        if (el.result === 'FAIL') {
            failures += logSymbols.error + ' ' + el.heading + '\n';
            failures += el.elements + '\n\n';
        }
    });

    grunt.log.writeln(indent(failures, ' ', 2));
    grunt.log.writeln(indent(passes  , ' ', 2));

    if (verbose) {
      grunt.log.writeln('VERBOSE OUTPUT\n', reports.audit);
    }

    return !failures.length;
  }
1143621
def CountFlowOutputPluginLogEntries(self,
                                      client_id,
                                      flow_id,
                                      output_plugin_id,
                                      with_type=None):
    """"""

    return len(
        self.ReadFlowOutputPluginLogEntries(
            client_id,
            flow_id,
            output_plugin_id,
            0,
            sys.maxsize,
            with_type=with_type))
930402
def j_count_plus(self, slug):
        '''
        
        '''
        output = {'status': 1 if MWiki.view_count_plus(slug) else 0}
        return json.dump(output, self)
892440
def ReadHuntOutputPluginLogEntries(self,
                                     hunt_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None,
                                     cursor=None):
    """"""
    query = ("SELECT client_id, flow_id, log_entry_type, message, "
             "UNIX_TIMESTAMP(timestamp) "
             "FROM flow_output_plugin_log_entries "
             "FORCE INDEX (flow_output_plugin_log_entries_by_hunt) "
             "WHERE hunt_id = %s AND output_plugin_id = %s ")
    args = [
        db_utils.HuntIDToInt(hunt_id),
        db_utils.OutputPluginIDToInt(output_plugin_id)
    ]

    if with_type is not None:
      query += "AND log_entry_type = %s "
      args.append(int(with_type))

    query += "ORDER BY log_id ASC LIMIT %s OFFSET %s"
    args.append(count)
    args.append(offset)

    cursor.execute(query, args)

    ret = []
    for (client_id_int, flow_id_int, log_entry_type, message,
         timestamp) in cursor.fetchall():
      ret.append(
          rdf_flow_objects.FlowOutputPluginLogEntry(
              hunt_id=hunt_id,
              client_id=db_utils.IntToClientID(client_id_int),
              flow_id=db_utils.IntToFlowID(flow_id_int),
              output_plugin_id=output_plugin_id,
              log_entry_type=log_entry_type,
              message=message,
              timestamp=mysql_utils.TimestampToRDFDatetime(timestamp)))

    return ret