Read/calculate hunt client resources stats.
555425
def ReadHuntClientResourcesStats(self, hunt_id, cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = """
      SELECT
        COUNT(*),
        SUM(user_cpu_time_used_micros),
        SUM((user_cpu_time_used_micros) * (user_cpu_time_used_micros)),
        SUM(system_cpu_time_used_micros),
        SUM((system_cpu_time_used_micros) * (system_cpu_time_used_micros)),
        SUM(network_bytes_sent),
        SUM(network_bytes_sent * network_bytes_sent),
    """

    scaled_bins = [
        int(1000000 * b) for b in rdf_stats.ClientResourcesStats.CPU_STATS_BINS
    ]

    query += self._BinsToQuery(scaled_bins, "(user_cpu_time_used_micros)")
    query += ","
    query += self._BinsToQuery(scaled_bins, "(system_cpu_time_used_micros)")
    query += ","
    query += self._BinsToQuery(
        rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS, "network_bytes_sent")

    query += " FROM flows "
    query += "FORCE INDEX(flows_by_hunt) "
    query += "WHERE parent_hunt_id = %s AND parent_flow_id IS NULL"

    cursor.execute(query, [hunt_id_int])

    response = cursor.fetchone()
    (count, user_sum, user_sq_sum, system_sum, system_sq_sum, network_sum,
     network_sq_sum) = response[:7]

    stats = rdf_stats.ClientResourcesStats(
        user_cpu_stats=rdf_stats.RunningStats(
            num=count,
            sum=db_utils.MicrosToSeconds(int(user_sum or 0)),
            sum_sq=int(user_sq_sum or 0) / 1e12,
        ),
        system_cpu_stats=rdf_stats.RunningStats(
            num=count,
            sum=db_utils.MicrosToSeconds(int(system_sum or 0)),
            sum_sq=int(system_sq_sum or 0) / 1e12,
        ),
        network_bytes_sent_stats=rdf_stats.RunningStats(
            num=count,
            sum=float(network_sum or 0),
            sum_sq=float(network_sq_sum or 0),
        ),
    )

    offset = 7
    stats.user_cpu_stats.histogram = rdf_stats.StatsHistogram()
    for b_num, b_max_value in zip(
        response[offset:], rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
      stats.user_cpu_stats.histogram.bins.append(
          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))

    offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
    stats.system_cpu_stats.histogram = rdf_stats.StatsHistogram()
    for b_num, b_max_value in zip(
        response[offset:], rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
      stats.system_cpu_stats.histogram.bins.append(
          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))

    offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
    stats.network_bytes_sent_stats.histogram = rdf_stats.StatsHistogram()
    for b_num, b_max_value in zip(
        response[offset:], rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS):
      stats.network_bytes_sent_stats.histogram.bins.append(
          rdf_stats.StatsHistogramBin(range_max_value=b_max_value, num=b_num))

    query = """
      SELECT
        client_id, flow_id, user_cpu_time_used_micros,
        system_cpu_time_used_micros, network_bytes_sent
      FROM flows
      FORCE INDEX(flows_by_hunt)
      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
            (user_cpu_time_used_micros > 0 OR
             system_cpu_time_used_micros > 0 OR
             network_bytes_sent > 0)
      ORDER BY (user_cpu_time_used_micros + system_cpu_time_used_micros) DESC
      LIMIT 10
    """

    cursor.execute(query, [hunt_id_int])

    for cid, fid, ucpu, scpu, nbs in cursor.fetchall():
      client_id = db_utils.IntToClientID(cid)
      flow_id = db_utils.IntToFlowID(fid)
      stats.worst_performers.append(
          rdf_client_stats.ClientResources(
              client_id=client_id,
              session_id=rdfvalue.RDFURN(client_id).Add(flow_id),
              cpu_usage=rdf_client_stats.CpuSeconds(
                  user_cpu_time=db_utils.MicrosToSeconds(ucpu),
                  system_cpu_time=db_utils.MicrosToSeconds(scpu),
              ),
              network_bytes_sent=nbs))

    return stats
1039269
def ReadClientLastPings(self,
                          min_last_ping=None,
                          max_last_ping=None,
                          fleetspeak_enabled=None):
    """"""
    last_pings = {}
    for client_id, metadata in iteritems(self.metadatas):
      last_ping = metadata.get("ping", rdfvalue.RDFDatetime(0))
      is_fleetspeak_client = metadata.get("fleetspeak_enabled", False)
      if min_last_ping is not None and last_ping < min_last_ping:
        continue
      elif max_last_ping is not None and last_ping > max_last_ping:
        continue
      elif (fleetspeak_enabled is not None and
            is_fleetspeak_client != fleetspeak_enabled):
        continue
      else:
        last_pings[client_id] = metadata.get("ping", None)
    return last_pings
580766
def check_resources(self, number):
    ''''''
    if not current_app.config.get('LINKCHECKING_ENABLED'):
        log.error('Link checking is disabled.')
        return

    base_pipeline = [
        {'$match': {'resources': {'$gt': []}}},
        {'$project': {'resources._id': True,
                      'resources.extras.check:date': True}},
        {'$unwind': '$resources'},
    ]
    # unchecked resources
    pipeline = base_pipeline + [
        {'$match': {'resources.extras.check:date': {'$eq': None}}},
        {'$limit': number}
    ]
    resources = list(Dataset.objects.aggregate(*pipeline))
    # not recently checked resources
    slots_left = number - len(resources)
    if slots_left:
        pipeline = base_pipeline + [
            {'$match': {'resources.extras.check:date': {'$ne': None}}},
            {'$sort': {'resources.extras.check:date': 1}},
            {'$limit': slots_left}
        ]
        resources += list(Dataset.objects.aggregate(*pipeline))

    nb_resources = len(resources)
    log.info('Checking %s resources...', nb_resources)
    for idx, dataset_resource in enumerate(resources):
        dataset_obj = Dataset.objects.get(id=dataset_resource['_id'])
        resource_id = dataset_resource['resources']['_id']
        rid = uuid.UUID(resource_id)
        resource_obj = get_by(dataset_obj.resources, 'id', rid)
        log.info('Checking resource %s (%s/%s)',
                 resource_id, idx + 1, nb_resources)
        if resource_obj.need_check():
            check_resource(resource_obj)
        else:
            log.info("--> Skipping this resource, cache is fresh enough.")
    log.info('Done.')
1120952
def ReadHuntResults(self,
                      hunt_id,
                      offset,
                      count,
                      with_tag=None,
                      with_type=None,
                      with_substring=None,
                      with_timestamp=None,
                      cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = ("SELECT client_id, flow_id, hunt_id, payload, type, "
             "UNIX_TIMESTAMP(timestamp), tag "
             "FROM flow_results "
             "FORCE INDEX(flow_results_hunt_id_flow_id_timestamp) "
             "WHERE hunt_id = %s ")

    args = [hunt_id_int]

    if with_tag:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type:
      query += "AND type = %s "
      args.append(with_type)

    if with_substring:
      query += "AND payload LIKE %s "
      args.append("%" + db_utils.EscapeWildcards(with_substring) + "%")

    if with_timestamp:
      query += "AND timestamp = FROM_UNIXTIME(%s) "
      args.append(mysql_utils.RDFDatetimeToTimestamp(with_timestamp))

    query += "ORDER BY timestamp ASC LIMIT %s OFFSET %s"
    args.append(count)
    args.append(offset)

    cursor.execute(query, args)

    ret = []
    for (
        client_id_int,
        flow_id_int,
        hunt_id_int,
        serialized_payload,
        payload_type,
        timestamp,
        tag,
    ) in cursor.fetchall():
      if payload_type in rdfvalue.RDFValue.classes:
        payload = rdfvalue.RDFValue.classes[payload_type]()
        payload.ParseFromString(serialized_payload)
      else:
        payload = rdf_objects.SerializedValueOfUnrecognizedType(
            type_name=payload_type, value=serialized_payload)

      result = rdf_flow_objects.FlowResult(
          client_id=db_utils.IntToClientID(client_id_int),
          flow_id=db_utils.IntToFlowID(flow_id_int),
          hunt_id=hunt_id,
          payload=payload,
          timestamp=mysql_utils.TimestampToRDFDatetime(timestamp))
      if tag is not None:
        result.tag = tag

      ret.append(result)

    return ret
828645
def ReadHuntOutputPluginLogEntries(self,
                                     hunt_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """"""

    all_entries = []
    for flow_obj in self._GetHuntFlows(hunt_id):
      for entry in self.ReadFlowOutputPluginLogEntries(
          flow_obj.client_id,
          flow_obj.flow_id,
          output_plugin_id,
          0,
          sys.maxsize,
          with_type=with_type):
        all_entries.append(
            rdf_flow_objects.FlowOutputPluginLogEntry(
                hunt_id=hunt_id,
                client_id=flow_obj.client_id,
                flow_id=flow_obj.flow_id,
                output_plugin_id=output_plugin_id,
                log_entry_type=entry.log_entry_type,
                timestamp=entry.timestamp,
                message=entry.message))

    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]
11247
def ProcessHuntFlowLog(flow_obj, log_msg):
  """"""

  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):
    return

  hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id)
  flow_urn = hunt_urn.Add(flow_obj.flow_id)
  log_entry = rdf_flows.FlowLog(
      client_id=flow_obj.client_id,
      urn=flow_urn,
      flow_name=flow_obj.flow_class_name,
      log_message=log_msg)
  with data_store.DB.GetMutationPool() as pool:
    grr_collections.LogCollection.StaticAdd(
        hunt_urn.Add("Logs"), log_entry, mutation_pool=pool)
664509
function() {
      var timings = {};
      var requests = win.performance.getEntriesByType("resource");
      for (var i = 0; i < requests.length; i++)
        timings[requests[i].name] = requests[i].responseEnd;
      for (var j = 0; j < rects.length; j++) {
        if (!('tm' in rects[j]))
          rects[j].tm = timings[rects[j].url] !== undefined ? timings[rects[j].url] : 0;
      }
    }
458058
def get_resources_count(self, link):
        
        total = 0
        for model in SupportedServices.get_service_resources(link.service):
            # Format query path from resource to service project link
            query = {model.Permissions.project_path.split('__')[0]: link}
            total += model.objects.filter(**query).count()
        return total
1222839
def retrieveVals(self):
        """"""
        lighttpdInfo = LighttpdInfo(self._host, self._port,
                                self._user, self._password, 
                                self._statuspath, self._ssl)
        stats = lighttpdInfo.getServerStats()
        if self.hasGraph('lighttpd_access'):
            self.setGraphVal('lighttpd_access', 'reqs', stats['Total Accesses'])
        if self.hasGraph('lighttpd_bytes'):
            self.setGraphVal('lighttpd_bytes', 'bytes', 
                             stats['Total kBytes'] * 1000)
        if self.hasGraph('lighttpd_servers'):
            self.setGraphVal('lighttpd_servers', 'busy', stats['BusyServers'])
            self.setGraphVal('lighttpd_servers', 'idle', stats['IdleServers'])
            self.setGraphVal('lighttpd_servers', 'max', stats['MaxServers'])
585932
def _HandleLegacy(self, args, token=None):
    """"""
    hunt_urn = args.hunt_id.ToURN()
    hunt_obj = aff4.FACTORY.Open(
        hunt_urn, aff4_type=implementation.GRRHunt, token=token)

    clients_by_status = hunt_obj.GetClientsByStatus()
    hunt_clients = clients_by_status[args.client_status.name]
    total_count = len(hunt_clients)

    if args.count:
      hunt_clients = sorted(hunt_clients)[args.offset:args.offset + args.count]
    else:
      hunt_clients = sorted(hunt_clients)[args.offset:]

    flow_id = "%s:hunt" % hunt_urn.Basename()
    results = [
        ApiHuntClient(client_id=c.Basename(), flow_id=flow_id)
        for c in hunt_clients
    ]
    return ApiListHuntClientsResult(items=results, total_count=total_count)
122757
def _get_total_read_size(self):
        """"""
        if self.read_size:
            read_size = EVENT_SIZE * self.read_size
        else:
            read_size = EVENT_SIZE
        return read_size
1016734
def CountHuntResults(self,
                       hunt_id,
                       with_tag=None,
                       with_type=None,
                       cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = "SELECT COUNT(*) FROM flow_results WHERE hunt_id = %s "

    args = [hunt_id_int]

    if with_tag is not None:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type is not None:
      query += "AND type = %s "
      args.append(with_type)

    cursor.execute(query, args)
    return cursor.fetchone()[0]
1226863
def ReadHuntFlows(self,
                    hunt_id,
                    offset,
                    count,
                    filter_condition=db.HuntFlowsCondition.UNSET):
    """"""
    if filter_condition == db.HuntFlowsCondition.UNSET:
      filter_fn = lambda _: True
    elif filter_condition == db.HuntFlowsCondition.FAILED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.ERROR
    elif filter_condition == db.HuntFlowsCondition.SUCCEEDED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.FINISHED
    elif filter_condition == db.HuntFlowsCondition.COMPLETED_FLOWS_ONLY:
      filter_fn = (
          lambda f: f.flow_state in [f.FlowState.ERROR, f.FlowState.FINISHED])
    elif filter_condition == db.HuntFlowsCondition.FLOWS_IN_PROGRESS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.RUNNING
    elif filter_condition == db.HuntFlowsCondition.CRASHED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.CRASHED
    else:
      raise ValueError("Invalid filter condition: %d" % filter_condition)

    results = [
        flow_obj for flow_obj in self._GetHuntFlows(hunt_id)
        if filter_fn(flow_obj)
    ]
    results.sort(key=lambda f: f.last_update_time)
    return results[offset:offset + count]
962960
def CountHuntResultsByType(self, hunt_id, cursor=None):
    """"""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = ("SELECT type, COUNT(*) FROM flow_results "
             "WHERE hunt_id = %s GROUP BY type")

    cursor.execute(query, [hunt_id_int])
    return dict(cursor.fetchall())
22910
def read(self, params, args, data):
        """"""
        result_count = None
        result_links = None

        if params is None:
            params = []
        if args:
            args = args.copy()
        else:
            args = {}

        ctx = self._create_context(params, args, data)
        row_id = ctx.get_row_id()

        if not row_id:
            self._transform_list_args(args)

            if 'page' in args or 'limit' in args:
                ctx = self._create_context(params, args, data)

                result_count = self._get_collection_count(ctx)

            if 'page' in args:
                result_links = _get_link_pages(
                    page=args['page'],
                    per_page=int(args['limit']),
                    count=result_count,
                    page_url=request.url
                )

            if 'limit' not in args:
                args['limit'] = 'unlimited'

        self._create_related(args)

        try:
            return ResourceResult(
                body=super(JsonServerResource, self).read(params, args, data),
                count=result_count,
                links=result_links
            )
        except NoResultFound:
            return NOT_FOUND
508926
@Override
    public ResourceSet<TaskQueuesStatistics> read(final TwilioRestClient client) {
        return new ResourceSet<>(this, client, firstPage(client));
    }
334069
def retrieveVals(self):
        """"""
        if self.hasGraph('tomcat_memory'):
            stats = self._tomcatInfo.getMemoryStats()
            self.setGraphVal('tomcat_memory', 'used', 
                             stats['total'] - stats['free'])
            self.setGraphVal('tomcat_memory', 'free', stats['free'])
            self.setGraphVal('tomcat_memory', 'max', stats['max'])
        for (port, stats) in self._tomcatInfo.getConnectorStats().iteritems():
            thrstats = stats['threadInfo']
            reqstats = stats['requestInfo']
            if self.portIncluded(port):
                name = "tomcat_threads_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'busy', 
                                     thrstats['currentThreadsBusy'])
                    self.setGraphVal(name, 'idle', 
                                     thrstats['currentThreadCount'] 
                                     - thrstats['currentThreadsBusy'])
                    self.setGraphVal(name, 'max', thrstats['maxThreads'])
                name = "tomcat_access_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'reqs', reqstats['requestCount'])
                name = "tomcat_error_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'errors', reqstats['errorCount'])
                name = "tomcat_traffic_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'rx', reqstats['bytesReceived'])
                    self.setGraphVal(name, 'tx', reqstats['bytesSent'])