/* (non-Javadoc)
@see org.duracloud.mill.ltp.LoopingTaskProducer#logCumulativeSessionStats()
552852
@Override
    protected void logCumulativeSessionStats(Map<String, RunStats> runstats, RunStats cumulativeTotals) {
        DuplicationRunStats dCumulativeTotals = (DuplicationRunStats) cumulativeTotals;
        log.info("session stats (global cumulative): domains={} dups={}  deletes={}",
                 runstats.keySet().size(), dCumulativeTotals.getDups(), dCumulativeTotals.getDeletes());

    }
848795
def statsAndLoggingAggregator(cls, jobStore, stop, config):
        
        #  Overall timing
        startTime = time.time()
        startClock = getTotalCpuTime()

        def callback(fileHandle):
            statsStr = fileHandle.read()
            if not isinstance(statsStr, str):
                statsStr = statsStr.decode()
            stats = json.loads(statsStr, object_hook=Expando)
            try:
                logs = stats.workers.logsToMaster
            except AttributeError:
                # To be expected if there were no calls to logToMaster()
                pass
            else:
                for message in logs:
                    logger.log(int(message.level),
                               'Got message from job at time %s: %s',
                               time.strftime('%m-%d-%Y %H:%M:%S'), message.text)
            try:
                logs = stats.logs
            except AttributeError:
                pass
            else:
                # we may have multiple jobs per worker
                jobNames = logs.names
                messages = logs.messages
                cls.logWithFormatting(jobNames[0], messages,
                                      message='Received Toil worker log. Disable debug level logging to hide this output')
                cls.writeLogFiles(jobNames, messages, config=config)

        while True:
            # This is a indirect way of getting a message to the thread to exit
            if stop.is_set():
                jobStore.readStatsAndLogging(callback)
                break
            if jobStore.readStatsAndLogging(callback) == 0:
                time.sleep(0.5)  # Avoid cycling too fast

        # Finish the stats file
        text = json.dumps(dict(total_time=str(time.time() - startTime),
                               total_clock=str(getTotalCpuTime() - startClock)), ensure_ascii=True)
        jobStore.writeStatsAndLogging(text)
576117
public void setLogDestinationConfigs(java.util.Collection<String> logDestinationConfigs) {
        if (logDestinationConfigs == null) {
            this.logDestinationConfigs = null;
            return;
        }

        this.logDestinationConfigs = new java.util.ArrayList<String>(logDestinationConfigs);
    }
1183696
@Override
  public synchronized void initTasks() throws IOException {
    boolean loggingEnabled = LOG.isDebugEnabled();
    if (loggingEnabled) {
      LOG.debug("(initTasks@SJIP) Starting Initialization for " + jobId);
    }
    numMapTasks = jobStory.getNumberMaps();
    numReduceTasks = jobStory.getNumberReduces();

    JobHistory.JobInfo.logSubmitted(getJobID(), conf, jobFile.toString(),
                                    this.startTime, hasRestarted());
    if (loggingEnabled) {
      LOG.debug("(initTasks@SJIP) Logged to job history for " + jobId);
    }

//    checkTaskLimits();

    if (loggingEnabled) {
      LOG.debug("(initTasks@SJIP) Checked task limits for " + jobId);
    }

    final String jobFile = "default";
    splits = getRawSplits(jobStory.getInputSplits());
    if (loggingEnabled) {
      LOG.debug("(initTasks@SJIP) Created splits for job = " + jobId
          + " number of splits = " + splits.length);
    }

//    createMapTasks(jobFile, splits);

    numMapTasks = splits.length;
    maps = new TaskInProgress[numMapTasks];
    for (int i=0; i < numMapTasks; ++i) {
      inputLength += splits[i].getDataLength();
      maps[i] = new TaskInProgress(jobId, jobFile,
                                   splits[i],
                                   conf, this, i, numSlotsPerMap);
    }    
    if (numMapTasks > 0) {
      nonRunningMapCache = createCache(splits, maxLevel);
      if (loggingEnabled) {
        LOG.debug("initTasks:numMaps=" + numMapTasks
            + " Size of nonRunningMapCache=" + nonRunningMapCache.size()
            + " for " + jobId);
      }
    }

    // set the launch time
    this.launchTime = JobTracker.getClock().getTime();

//    createReduceTasks(jobFile);

    //   
    // Create reduce tasks
    //   
    this.reduces = new TaskInProgress[numReduceTasks];
    for (int i = 0; i < numReduceTasks; i++) {
      reduces[i] = new TaskInProgress(jobId, jobFile,
                                      numMapTasks, i,
                                      conf, this, numSlotsPerReduce);
      nonRunningReduces.add(reduces[i]);
    }    

    // Calculate the minimum number of maps to be complete before
    // we should start scheduling reduces
    completedMapsForReduceSlowstart = (int) Math.ceil((conf.getFloat(
        "mapred.reduce.slowstart." + "completed.maps",
        DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART) * numMapTasks));

    tasksInited.set(true);
    if (loggingEnabled) {
      LOG.debug("Initializing job, nowstatus = "
          + JobStatus.getJobRunState(getStatus().getRunState()));
    }
    setupComplete();

    if (loggingEnabled) {
      LOG.debug("Initializing job, inited-status = "
          + JobStatus.getJobRunState(getStatus().getRunState()));
    }
  }
552974
protected void logVarstat(DoubleStatistic varstat, double[] varsum) {
    if(varstat != null) {
      double s = sum(varsum);
      getLogger().statistics(varstat.setDouble(s));
    }
  }
1046322
public static void log(Marker marker, PerformanceMetrics metrics) {
    if (log.isDebugEnabled()) {
      log.debug(marker, getEndMetrics(metrics));
    }
  }
1223454
@Warmup(iterations = 5)
    @Measurement(iterations = 5)
    @Benchmark
    public void logstashStructuredParametrizedMessageLogging1Call() {
        loggerLogstashParametrizedMessage.info("Event with double={} and boolean={}", value("varDouble", 1.2), value("varBoolean", false));
    }
1086987
@SuppressLint("InvalidAccessToGuardedField")
  private void logStats() {
    if (FLog.isLoggable(FLog.VERBOSE)) {
      FLog.v(
          TAG,
          "Used = (%d, %d); Free = (%d, %d)",
          mUsed.mCount,
          mUsed.mNumBytes,
          mFree.mCount,
          mFree.mNumBytes);
    }
  }
27989
private void log(ArrayList<PrintWriter> writers, RecordTypes recordType,
                   Keys[] keys, String[] values) {
    log(writers, recordType, keys, values, false);
  }
1003224
public static void log(final String templateName, final Template template, final Values values) {
        new ValuesLogger().doLog(templateName, template, values);
    }
177777
private void processLoggingEvents(List<LoggingEvent> loggingEvents) {
        // Lazy initialization of the appender. This is needed because the
        // original appenders configuration may be available only after the
        // complete
        // log4j initialization.
        while (appenders.getAllAppenders() == null) {
            if ((batcher == null) || (batcher.isPaused())) {
                try {
                    Thread.sleep(SLEEP_TIME_MS);
                } catch (InterruptedException ignore) {

                }
                continue;
            }

            org.apache.log4j.Logger asyncLogger = LoggerCache.getInstance()
                    .getOrCreateLogger(LOGGER_ASYNC_APPENDER);
            Appender originalAppender = asyncLogger
                    .getAppender(originalAppenderName);
            if (originalAppender == null) {
                try {
                    Thread.sleep(SLEEP_TIME_MS);
                } catch (InterruptedException ignore) {

                }
                continue;
            }
            appenders.addAppender(originalAppender);
        }
        // First take the overflown summary events and put it back in the queue
        for (Iterator<Entry<String, LogSummary>> iter = logSummaryMap
                .entrySet().iterator(); iter.hasNext();) {
            Entry<String, LogSummary> mapEntry = (Entry<String, LogSummary>) iter
                    .next();
            // If the space is not available, then exit immediately
            if (batcher.isSpaceAvailable()) {
                LogSummary logSummary = mapEntry.getValue();
                LoggingEvent event = logSummary.createEvent();
                // Put the event in the queue and remove the event from the summary
                if (batcher.process(event)) {
                    iter.remove();
                } else {
                    break;
                }
            } else {
                break;
            }

        }
        // Process the events from the queue and call the underlying
        // appender
        for (LoggingEvent event : loggingEvents) {
            appenders.appendLoopOnAppenders(event);

        }

    }
375259
@Override
    protected void logGlobalncrementalStats(RunStats incrementalTotals) {
        DuplicationRunStats dIncrementalTotals = (DuplicationRunStats) incrementalTotals;

        log.info("Session stats (global incremental): dups={} deletes={}",
                 dIncrementalTotals.getDups(), dIncrementalTotals.getDeletes());

    }
966636
public static void logMeasures(HLAMeasure[] measures, final Logger log) {

        log.info("\nAvailable measures are:");
        log.info("======================");

        for (HLAMeasure currMeasure : measures) {

            log.info(" - " + currMeasure.getSonarName());
        }

        LogHelper.moo(log);
    }
699157
private void logResults() {
    LOG.info("Key Values Processed: " + kvs_processed.get());
    LOG.info("Rows Processed: " + rows_processed.get());
    LOG.info("Valid Datapoints: " + valid_datapoints.get());
    LOG.info("Annotations: " + annotations.get());
    LOG.info("Invalid Row Keys Found: " + bad_key.get());
    LOG.info("Invalid Rows Deleted: " + bad_key_fixed.get());
    LOG.info("Duplicate Datapoints: " + duplicates.get());
    LOG.info("Duplicate Datapoints Resolved: " + duplicates_fixed.get());
    LOG.info("Orphaned UID Rows: " + orphans.get());
    LOG.info("Orphaned UID Rows Deleted: " + orphans_fixed.get());
    LOG.info("Possible Future Objects: " + future.get());
    LOG.info("Unknown Objects: " + unknown.get());
    LOG.info("Unknown Objects Deleted: " + unknown_fixed.get());
    LOG.info("Unparseable Datapoint Values: " + bad_values.get());
    LOG.info("Unparseable Datapoint Values Deleted: " + bad_values_deleted.get());
    LOG.info("Improperly Encoded Floating Point Values: " + value_encoding.get());
    LOG.info("Improperly Encoded Floating Point Values Fixed: " +
        value_encoding_fixed.get());
    LOG.info("Unparseable Compacted Columns: " + bad_compacted_columns.get());
    LOG.info("Unparseable Compacted Columns Deleted: " +
        bad_compacted_columns_deleted.get());
    LOG.info("Datapoints Qualified for VLE : " + vle.get());
    LOG.info("Datapoints Compressed with VLE: " + vle_fixed.get());
    LOG.info("Bytes Saved with VLE: " + vle_bytes.get());
    LOG.info("Total Errors: " + totalErrors());
    LOG.info("Total Correctable Errors: " + correctable());
    LOG.info("Total Errors Fixed: " + totalFixed());
  }
1120689
@SuppressWarnings("unchecked")
    private void processJavaLoggerDiscovery(ClassTraceInfo info) {
        List<FieldNode> loggerFields = getFieldsByDesc(LOGGER_TYPE.getDescriptor(), info.classNode.fields);
        if (!loggerFields.isEmpty()) {
            // Remove references to non-static Loggers
            for (int i = loggerFields.size() - 1; i >= 0; i--) {
                FieldNode fn = loggerFields.get(i);
                if ((fn.access & Opcodes.ACC_STATIC) != Opcodes.ACC_STATIC) {
                    loggerFields.remove(i);
                    StringBuilder sb = new StringBuilder();
                    sb.append("WARNING: Non-static java.util.logging.Logger field declared on class ");
                    sb.append(info.classNode.name.replaceAll("/", "\\.")).append(": ");
                    sb.append(fn.name);
                    info.warnings.add(sb.toString());
                }
            }
            if (loggerFields.size() > 1) {
                StringBuilder sb = new StringBuilder();
                sb.append("WARNING: Multiple java.util.logging.Logger fields declared on class ");
                sb.append(info.classNode.name.replaceAll("/", "\\.")).append(": ");
                for (int i = 0; i < loggerFields.size(); i++) {
                    sb.append(loggerFields.get(i).name);
                    if (i + 1 != loggerFields.size()) {
                        sb.append(", ");
                    }
                }
                info.warnings.add(sb.toString());
            }

            // Keep track of the first static Logger
            if (loggerFields.size() > 0) {
                info.loggerFieldNode = loggerFields.get(0);
            }
        }
    }
36010
protected java.util.Map captureStatistics()
    {
        if (Tracing.isAnyTracingEnabled() && trace.isEntryEnabled())
            trace.entry(this, cclass
                        , "captureStatistics"
                            );

        java.util.Map statistics = super.captureStatistics();

        statistics.put("totalBytesWritten", Long.toString(totalBytesWritten));
        statistics.put("totalNumberOfFlushRequests", Long.toString(totalNumberOfFlushRequests));
        statistics.put("totalNumberOfLogBufferWrites", Long.toString(totalNumberOfLogBufferWrites));
        statistics.put("totalNumberOfThreadsFindingFullLogBuffers", Long.toString(totalNumberOfThreadsFindingFullLogBuffers));
        statistics.put("totalNumberOfFlusHelperWaits", Long.toString(totalNumberOfFlushHelperWaits));
        statistics.put("totalNumberOfLogCycles", Long.toString(totalNumberOfLogCycles));

        statistics.put("writeStalledMilliseconds", Long.toString(writeStalledMilliseconds));
        statistics.put("writeCopyingMilliseconds", Long.toString(writeCopyingMilliseconds));

        statistics.put("flushingMilliseconds", Long.toString(flushingMilliseconds));
        statistics.put("otherMilliseconds", Long.toString(otherMilliseconds));
        statistics.put("flushHelperWaitingMilliseconds", Long.toString(flushHelperWaitingMilliseconds));
        statistics.put("paddingStalledMilliseconds", Long.toString(paddingStalledMilliseconds));
        statistics.put("writeUpdateStatedMilliseconds", Long.toString(writeUpdateStateMilliseconds));

        statistics.put("logFullCheckpointsTriggered", Long.toString(logFullCheckpointsTriggered));
        statistics.put("uncheckedBytes", Long.toString(uncheckedBytes));
        statistics.put("uncheckedBytesUpToTruncatePoint", Long.toString(uncheckedBytesUpToMarkPoint));
        statistics.put("getLogFileSpaceLeft()", Long.toString(getLogFileSpaceLeft()));
        statistics.put("fileLogHeader.fileSize", Long.toString(fileLogHeader.fileSize));
        statistics.put("bufferFilePosition+nextFreeByteInLogBuffer", Long.toString(bufferFilePosition + nextFreeByteInLogBuffer));
        statistics.put("stalledForMultiPartID", Long.toString(stalledForMultiPartID));
        statistics.put("totalPaddingBytesWritten", Long.toString(totalPaddingBytesWritten));
        statistics.put("totalPaddingRecords", Long.toString(totalPaddingRecords));

        String histogram = "(0-15 >15) ";
        for (int n = 0; n < numberOfFlushWaitersFrequency.length; n++) {
            histogram += numberOfFlushWaitersFrequency[n] + " ";
            numberOfFlushWaitersFrequency[n] = 0;
        }
        statistics.put("numberOfFlushWaitersFrequency", histogram);

        histogram = "(0-15 >15) ";
        for (int n = 0; n < numberOfPagesWrittenFrequency.length; n++) {
            histogram += numberOfPagesWrittenFrequency[n] + " ";
            numberOfPagesWrittenFrequency[n] = 0;
        }
        statistics.put("numberOfPagesWritttenFrequency", histogram);

        totalBytesWritten = 0; // Number of bytes written so far.
        totalNumberOfFlushRequests = 0; // Number of times flush() was invoked.
        totalNumberOfLogBufferWrites = 0; // Number of times the logBuffer is has written.
        totalNumberOfThreadsFindingFullLogBuffers = 0;
        totalNumberOfFlushHelperWaits = 0; // Number of times the FlushHelper had nothing to do.
        totalNumberOfLogCycles = 0; // Number of times we have cycled through the log.

        writeStalledMilliseconds = 0;
        writeCopyingMilliseconds = 0;
        flushingMilliseconds = 0;
        otherMilliseconds = 0;
        flushHelperWaitingMilliseconds = 0;
        paddingStalledMilliseconds = 0;
        writeUpdateStateMilliseconds = 0;
        stalledForMultiPartID = 0;
        totalPaddingBytesWritten = 0;
        totalPaddingRecords = 0;

        if (Tracing.isAnyTracingEnabled() && trace.isEntryEnabled())
            trace.exit(this, cclass
                       , "captureStatistics"
                       , new Object[] { statistics }
                            );
        return statistics;
    }
1244671
public String getResults(long totalErrors) {

		Object[] info = { fileCount, doneTasks.get(COMPILED).get(),
				startedTasks.get(COMPILED).get(),
				doneTasks.get(ANNOTATION).get(),
				startedTasks.get(ANNOTATION).get(), doneTasks.get(XML).get(),
				startedTasks.get(XML).get(), doneTasks.get(DEP).get(),
				startedTasks.get(DEP).get(), totalErrors, buildTime,
				convertToMB(heapUsed.get()), convertToMB(heapTotal.get()),
				convertToMB(nonHeapUsed.get()), convertToMB(nonHeapTotal.get()) };

		return MessageUtils.format(MSG_STATISTICS_TEMPLATE, info);
	}