Get the processing rate for this task (e.g. bytes/ms in reduce)
421568
public double getProcessingRate(TaskStatus.Phase phase) {
    // we don't have processing rate information for the starting and cleaning
    // up phase
    if (phase != TaskStatus.Phase.MAP && 
        phase != TaskStatus.Phase.SHUFFLE &&
        phase != TaskStatus.Phase.SORT &&
        phase != TaskStatus.Phase.REDUCE) {
      return 0;
    }
    return processingRates.getRate(getProcessingPhase());
  }
825420
def get_bulkrates(self):
        """"""
        try:
            resp = get(self.bulk_url)
            resp.raise_for_status()
        except exceptions.RequestException as e:
            raise RuntimeError(e)
        return resp.json()
1125819
private double computeShare(JobInfo info, double w2sRatio,
      TaskType type, boolean considerMinMax) {
    if (!isRunnable(info)) {
      return 0;
    }
    double share = type == TaskType.MAP ? info.mapWeight : info.reduceWeight;
    share *= w2sRatio;
    if (considerMinMax) {
      int minSlots = type == TaskType.MAP ? info.minMaps : info.minReduces;
      share = Math.max(share, minSlots);
      int maxSlots = type == TaskType.MAP ? info.maxMaps : info.maxReduces;
      share = Math.min(share, maxSlots);
    }
    share = Math.min(share, runnableTasks(info, type));
    return share;
  }
924082
def _PrintProcessingTime(self, processing_status):
    
    if not processing_status:
      processing_time = '00:00:00'
    else:
      processing_time = time.time() - processing_status.start_time
      time_struct = time.gmtime(processing_time)
      processing_time = time.strftime('%H:%M:%S', time_struct)

    self._output_writer.Write(
        'Processing time\t\t: {0:s}\n'.format(processing_time))
861274
def process_cpu_times(&callback)
      determine_cpu_times = Proc.new do
        ::Process.times.to_a rescue [nil, nil, nil, nil]
      end
      EM::defer(determine_cpu_times, callback)
    end
797894
def _emit_metrics(self, missing_datetimes, finite_start, finite_stop):
        
        datetimes = self.finite_datetimes(
            finite_start if self.start is None else min(finite_start, self.parameter_to_datetime(self.start)),
            finite_stop if self.stop is None else max(finite_stop, self.parameter_to_datetime(self.stop)))

        delay_in_jobs = len(datetimes) - datetimes.index(missing_datetimes[0]) if datetimes and missing_datetimes else 0
        self.trigger_event(RangeEvent.DELAY, self.of_cls.task_family, delay_in_jobs)

        expected_count = len(datetimes)
        complete_count = expected_count - len(missing_datetimes)
        self.trigger_event(RangeEvent.COMPLETE_COUNT, self.of_cls.task_family, complete_count)
        self.trigger_event(RangeEvent.COMPLETE_FRACTION, self.of_cls.task_family, float(complete_count) / expected_count if expected_count else 1)
1124513
def compute_file_metrics(processors, language, key, token_list):
    """"""
    # multiply iterator
    tli = itertools.tee(token_list, len(processors))
    metrics = OrderedDict()

    # reset all processors
    for p in processors:
        p.reset()

    # process all tokens
    for p, tl in zip(processors, tli):
        p.process_file(language, key, tl)

    # collect metrics from all processors
    for p in processors:
        metrics.update(p.metrics)

    return metrics
289648
def str_rate(self):
        """"""
        # Handle special cases.
        if not self._eta.started or self._eta.stalled or not self.rate:
            return '--.-KiB/s'

        unit_rate, unit = UnitByte(self._eta.rate_overall if self.done else self.rate).auto
        if unit_rate >= 100:
            formatter = '%d'
        elif unit_rate >= 10:
            formatter = '%.1f'
        else:
            formatter = '%.2f'
        return '{0}{1}/s'.format(locale.format(formatter, unit_rate, grouping=False), unit)
1191468
def count(self, task):
        
        return len([c for c in task.corrections if c["event"]["@class"] == self.event_class])
910971
def _get_result(future, pipe, timeout):
    """"""
    counter = count(step=SLEEP_UNIT)

    try:
        while not pipe.poll(SLEEP_UNIT):
            if timeout is not None and next(counter) >= timeout:
                return TimeoutError('Task Timeout', timeout)
            elif future.cancelled():
                return CancelledError()

        return pipe.recv()
    except (EOFError, OSError):
        return ProcessExpired('Abnormal termination')
    except Exception as error:
        return error
283358
def get_num_processes():
    """"""
    cpu_count = multiprocessing.cpu_count()

    if config.NUMBER_OF_CORES == 0:
        raise ValueError(
            'Invalid NUMBER_OF_CORES; value may not be 0.')

    if config.NUMBER_OF_CORES > cpu_count:
        log.info('Requesting %s cores; only %s available',
                 config.NUMBER_OF_CORES, cpu_count)
        return cpu_count

    if config.NUMBER_OF_CORES < 0:
        num = cpu_count + config.NUMBER_OF_CORES + 1
        if num <= 0:
            raise ValueError(
                'Invalid NUMBER_OF_CORES; negative value is too negative: '
                'requesting {} cores, {} available.'.format(num, cpu_count))

        return num

    return config.NUMBER_OF_CORES
1103739
def _mapreduce_task_metrics(self, running_jobs, auth, ssl_verify, addl_tags):
        
        for job_stats in itervalues(running_jobs):

            metrics_json = self._rest_request_to_json(
                job_stats['tracking_url'], auth, ssl_verify, 'tasks', self.MAPREDUCE_SERVICE_CHECK, tags=addl_tags
            )

            if metrics_json.get('tasks'):
                if metrics_json['tasks'].get('task'):

                    for task in metrics_json['tasks']['task']:
                        task_type = task.get('type')

                        if task_type:
                            tags = [
                                'app_name:' + job_stats['app_name'],
                                'user_name:' + job_stats['user_name'],
                                'job_name:' + job_stats['job_name'],
                                'task_type:' + str(task_type).lower(),
                            ]

                            tags.extend(addl_tags)

                            if task_type == 'MAP':
                                self._set_metrics_from_json(task, self.MAPREDUCE_MAP_TASK_METRICS, tags)

                            elif task_type == 'REDUCE':
                                self._set_metrics_from_json(task, self.MAPREDUCE_REDUCE_TASK_METRICS, tags)
740390
def avg_receive_rate(self):
        
        if not self._has_data or 'sum_received' not in self.result['end']:
            return None
        bps = self.result['end']['sum_received']['bits_per_second']
        return bps / 8 / 1024 / 1024
101309
def process_info(pid=None):
    '''
    '''
    if psutil is None:  # pragma    nocover
        return {}
    pid = pid or os.getpid()
    try:
        p = psutil.Process(pid)
    # this fails on platforms which don't allow multiprocessing
    except psutil.NoSuchProcess:  # pragma    nocover
        return {}
    else:
        mem = p.memory_info()
        return {'memory': convert_bytes(mem.rss),
                'memory_virtual': convert_bytes(mem.vms),
                'cpu_percent': p.cpu_percent(),
                'nice': p.nice(),
                'num_threads': p.num_threads()}
531918
function sync() {
    // retrieve latency, then wait 1 sec
    function getLatencyAndWait() {
      var result = null;

      if (isDestroyed) {
        return Promise.resolve(result);
      }

      return getLatency(slave)
          .then(function (latency) { result = latency })  // store the retrieved latency
          .catch(function (err)    { console.log(err) })  // just log failed requests
          .then(function () { return util.wait(DELAY) })  // wait 1 sec
          .then(function () { return result});            // return the retrieved latency
    }

    return util
        .repeat(getLatencyAndWait, REPEAT)
        .then(function (all) {
          debug('latencies', all);

          // filter away failed requests
          var latencies = all.filter(function (latency) {
            return latency !== null;
          });

          // calculate the limit for outliers
          var limit = stat.median(latencies) + stat.std(latencies);

          // filter away outliers: all latencies largereq than the mean+std
          var filtered = latencies.filter(function (latency) {
            return latency < limit;
          });

          // return the mean latency
          return (filtered.length > 0) ? stat.mean(filtered) : null;
        })
        .then(function (latency) {
          if (isDestroyed) {
            return Promise.resolve(null);
          }
          else {
            return slave.request('time').then(function (timestamp) {
              var time = timestamp + latency;
              slave.emit('change', time);
              return time;
            });
          }
        })
        .catch(function (err) {
          slave.emit('error', err)
        });
  }
158073
def process(self, metric):
        
        entry = {'timestamp': metric.timestamp, 'value': metric.value,
                 "tags": {}}
        entry["tags"]["hostname"] = metric.host

        if self.cleanMetrics:
            metric = MetricWrapper(metric, self.log)
            if self.skipAggregates and metric.isAggregate():
                return
            for tagKey in metric.getTags():
                entry["tags"][tagKey] = metric.getTags()[tagKey]

        entry['metric'] = (self.prefix + metric.getCollectorPath() +
                           '.' + metric.getMetricPath())

        for [key, value] in self.tags:
            entry["tags"][key] = value

        self.entrys.append(entry)

        # send data if list is long enough
        if (len(self.entrys) >= self.batch):
            # Compress data
            if self.compression >= 1:
                data = StringIO.StringIO()
                with contextlib.closing(gzip.GzipFile(fileobj=data,
                                        compresslevel=self.compression,
                                        mode="w")) as f:
                    f.write(json.dumps(self.entrys))
                self._send(data.getvalue())
            else:
                # no compression
                data = json.dumps(self.entrys)
                self._send(data)
318271
def avg_send_rate(self):
        
        if not self._has_data or 'sum_sent' not in self.result['end']:
            return None
        bps = self.result['end']['sum_sent']['bits_per_second']
        return bps / 8 / 1024 / 1024