Policy needed for logspout -> kinesis log streaming.
481892
def logstream_policy():
    """"""
    p = Policy(
        Statement=[
            Statement(
                Effect=Allow,
                Resource=["*"],
                Action=[
                    kinesis.CreateStream, kinesis.DescribeStream,
                    Action(kinesis.prefix, "AddTagsToStream"),
                    Action(kinesis.prefix, "PutRecords")
                ])])
    return p
697899
function LogstashStream(options) {
  EventEmitter.call(this);
  options = options || {};

  this.name = 'bunyan';
  this.level = options.level || 'info';
  this.server = options.server || os.hostname();
  this.host = options.host || '127.0.0.1';
  this.port = options.port || 9999;
  this.application = options.appName || process.title;
  this.pid = options.pid || process.pid;
  this.tags = options.tags || ['bunyan'];
  this.type = options.type;

  this.client = null;

  // ssl
  this.ssl_enable = options.ssl_enable || false;
  this.ssl_key = options.ssl_key || '';
  this.ssl_cert = options.ssl_cert || '';
  this.ca = options.ca || '';
  this.ssl_passphrase = options.ssl_passphrase || '';

  this.cbuffer_size = options.cbuffer_size || 10;

  // Connection state
  this.log_queue = new CBuffer(this.cbuffer_size);
  this.connected = false;
  this.socket = null;
  this.retries = -1;

  this.max_connect_retries = (typeof options.max_connect_retries === 'number') ? options.max_connect_retries : 4;
  this.retry_interval = options.retry_interval || 100;

  this.connect();
}
501810
def create_log_dict(request, response):
    
    remote_addr = request.META.get('REMOTE_ADDR')
    if remote_addr in getattr(settings, 'INTERNAL_IPS', []):
        remote_addr = request.META.get(
            'HTTP_X_FORWARDED_FOR') or remote_addr

    user_email = "-"
    if hasattr(request, 'user'):
        user_email = getattr(request.user, 'email', '-')

    if response.streaming:
        content_length = 'streaming'
    else:
        content_length = len(response.content)

    return {
        # 'event' makes event-based filtering possible in logging backends
        # like logstash
        'event': settings.LOGUTILS_LOGGING_MIDDLEWARE_EVENT,
        'remote_address': remote_addr,
        'user_email': user_email,
        'method': request.method,
        'url': request.get_full_path(),
        'status': response.status_code,
        'content_length': content_length,
        'request_time': -1,  # NA value: real value added by LoggingMiddleware
    }
675273
def _log_control(self, s):
        """"""
        if self.encoding is not None:
            s = s.decode(self.encoding, 'replace')
        self._log(s, 'send')
1212055
public void logBufferUtilization() {
		System.out.println("Buffer utilization at " + System.currentTimeMillis());

		System.out.println("\tUnused global buffers: " + this.globalBufferPool.numAvailableBuffers());

		System.out.println("\tLocal buffer pool status:");

		for (LocalBufferPoolOwner bufferPool : this.localBuffersPools.values()) {
			bufferPool.logBufferUtilization();
		}

		System.out.println("\tIncoming connections:");

		for (Channel channel : this.channels.values()) {
			if (channel.isInputChannel()) {
				((InputChannel<?>) channel).logQueuedEnvelopes();
			}
		}
	}
646188
private void logDescription(LoggingStrategy loggingStrategy, Rule rule) {
        String description = rule.getDescription();
        StringTokenizer tokenizer = new StringTokenizer(description, "\n");
        while (tokenizer.hasMoreTokens()) {
            loggingStrategy.log(tokenizer.nextToken().replaceAll("(\\r|\\n|\\t)", ""));
        }
    }
1315849
function Stream(log, connection) {
  Duplex.call(this);

  // * logging
  this._log = log.child({ component: 'stream', s: this });

  // * receiving and sending stream management commands
  this._initializeManagement();

  // * sending and receiving frames to/from the upstream connection
  this._initializeDataFlow();

  // * maintaining the state of the stream (idle, open, closed, etc.) and error detection
  this._initializeState();

  this.connection = connection;
}
1033562
def http_log_graph(self, stream):
        '''  '''
        print 'Entering http_log_graph...'
        for row in list(stream):

            # Skip '-' hosts
            if (row['id.orig_h'] == '-'):
                continue

            # Add the originating host
            self.add_node(row['id.orig_h'], row['id.orig_h'], ['host', 'origin'])

            # Add the response host and reponse ip
            self.add_node(row['host'], row['host'], ['host'])
            self.add_node(row['id.resp_h'], row['id.resp_h'], ['host'])

            # Add the http request relationships
            self.add_rel(row['id.orig_h'], row['host'], 'http_request')
            self.add_rel(row['host'], row['id.resp_h'], 'A')
1240709
def logs(ctx, services, num, follow):
    """"""
    logger.debug("running command %s (%s)", ctx.command.name, ctx.params,
                 extra={"command": ctx.command.name, "params": ctx.params})

    home = ctx.obj["HOME"]
    services_path = os.path.join(home, SERVICES)

    tail_threads = []
    for service in services:
        logpath = os.path.join(services_path, service, LOGS_DIR, STDOUTLOG)
        if os.path.exists(logpath):
            logger.debug("tailing %s", logpath)
            # TODO: Print log lines from multiple services sorted by timestamp
            t = threading.Thread(target=Tailer, kwargs={"name": service,
                                                        "nlines": num,
                                                        "filepath": logpath,
                                                        "follow": follow})
            t.daemon = True
            t.start()
            tail_threads.append(t)

    if tail_threads:
        while tail_threads[0].isAlive():
            tail_threads[0].join(0.1)
149811
def _log(self, message, stream, color=None, newline=False):
        '''
        
        '''

        if color and self.color_term:
            colorend = Logger.COLOR_END
        else:
            color = colorend = ''

        stream.write('{color}{message}{colorend}\n'.format(
            color=color,
            message=message,
            colorend=colorend
        ))

        if newline:
            sys.stdout.write('\n')

        stream.flush()
299040
function () {
            // The strategy to send logs is either after a certain delay, or when the stack is bigger than ...
            var now = new Date().getTime();
            if (this._logStack.length > this.minimumLogNb && now > this._lastLogSent + this.minimumInterval) {
                this._lastLogSent = new Date().getTime();
                this._sendStack();
                this._logStack = [];
            }
        }
708092
def logStream(fileHandle, shortName, printFunction=logger.info):
    
    printFunction("Reporting file: %s" % shortName)
    line = fileHandle.readline()
    while line != '':
        if line[-1] == '\n':
            line = line[:-1]
        printFunction("%s:\t%s" % (shortName, line))
        line = fileHandle.readline()
    fileHandle.close()
477476
function Connection(log, firstStreamId, settings) {
  // * initializing the base class
  Flow.call(this, 0);

  // * logging: every method uses the common logger object
  this._log = log.child({ component: 'connection' });

  // * stream management
  this._initializeStreamManagement(firstStreamId);

  // * lifecycle management
  this._initializeLifecycleManagement();

  // * flow control
  this._initializeFlowControl();

  // * settings management
  this._initializeSettingsManagement(settings);

  this._initializeConnectionFlowControl();

  // * multiplexing
  this._initializeMultiplexing();
}
1170996
function proceedWithBackup() {
    if (options.resume) {
      // pick up from existing log file from previous run
      downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
    } else {
      // create new log file and process
      spoolchanges(db, options.log, options.bufferSize, ee, function(err) {
        if (err) {
          ee.emit('error', err);
        } else {
          downloadRemainingBatches(options.log, db, ee, start, batchesPerDownloadSession, options.parallelism);
        }
      });
    }
  }
771140
def StreamMetrics(self, request_iterator, context):
        """"""
        LOG.debug("StreamMetrics called")

        # set up arguments
        collect_args = (next(request_iterator))
        max_metrics_buffer = 0
        max_collect_duration = 0
        cfg = Metric(pb=collect_args.Metrics_Arg.metrics[0])
        try:
            max_metrics_buffer = int(cfg.config["max-metrics-buffer"])
        except Exception as ex:
            LOG.debug("Unable to get schedule parameters: {}".format(ex))
        try:
            max_collect_duration = int(cfg.config["max-collect-duration"])
        except Exception as ex:
            LOG.debug("Unable to get schedule parameters: {}".format(ex))
        if max_metrics_buffer > 0:
            self.max_metrics_buffer = max_metrics_buffer
        if max_collect_duration > 0:
            self.max_collect_duration = max_collect_duration

        # start collection thread
        thread = threading.Thread(target=self._stream_wrapper, args=(collect_args,),)
        thread.daemon = True
        thread.start()

        # stream metrics
        metrics = []
        metrics_to_stream = []
        stream_timeout = self.max_collect_duration
        while context.is_active():
            try:
                # wait for metrics until timeout is reached
                t_start = time.time()
                metrics = self.metrics_queue.get(block=True, timeout=stream_timeout)
                elapsed = round(time.time() - t_start)
                stream_timeout -= elapsed
            except queue.Empty:
                LOG.debug("Max collect duration exceeded. Streaming {} metrics".format(len(metrics_to_stream)))
                metrics_col = CollectReply(Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                metrics_to_stream = []
                stream_timeout = self.max_collect_duration
                yield metrics_col
            else:
                for metric in metrics:
                    metrics_to_stream.append(metric)
                    if len(metrics_to_stream) == self.max_metrics_buffer:
                        LOG.debug("Max metrics buffer reached. Streaming {} metrics".format(len(metrics_to_stream)))
                        metrics_col = CollectReply(
                            Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                        metrics_to_stream = []
                        stream_timeout = self.max_collect_duration
                        yield metrics_col
                # stream metrics if max_metrics_buffer is 0 or enough metrics has been collected
                if self.max_metrics_buffer == 0:
                    LOG.debug("Max metrics buffer set to 0. Streaming {} metrics".format(len(metrics_to_stream)))
                    metrics_col = CollectReply(Metrics_Reply=MetricsReply(metrics=[m.pb for m in metrics_to_stream]))
                    metrics_to_stream = []
                    stream_timeout = self.max_collect_duration
                    yield metrics_col

        # sent notification if stream has been stopped
        self.done_queue.put(True)
301761
public function runCronArchiving()
    {
        Piwik::checkUserHasSuperUserAccess();

        // HTTP request: logs needs to be dumped in the HTTP response (on top of existing log destinations)
        /** @var \Monolog\Logger $logger */
        $logger = StaticContainer::get('Psr\Log\LoggerInterface');
        $handler = new StreamHandler('php://output', Logger::INFO);
        $handler->setFormatter(StaticContainer::get('Piwik\Plugins\Monolog\Formatter\LineMessageFormatter'));
        $logger->pushHandler($handler);

        $archiver = new CronArchive();
        $archiver->main();
    }
710067
void scheduleDailyLoggingWorkInNextCheckTime() {
        DailyRollingFileAppender dailyAppender = null;
        Enumeration<?> appenders = Logger.getRootLogger().getAllAppenders();
        while (appenders.hasMoreElements()) {
            Appender appender = (Appender) appenders.nextElement();
            if (appender instanceof DailyRollingFileAppender){
                dailyAppender = (DailyRollingFileAppender) appender;
            }
        }
        final DailyRollingFileAppender dailyRollingFileAppender = dailyAppender;

        Field field = null;
        if (dailyRollingFileAppender != null) {
            try {
                field = dailyRollingFileAppender.getClass().getDeclaredField("nextCheck");
                field.setAccessible(true);
            } catch (NoSuchFieldException e) {
                hostLog.error("Failed to set daily system info logging: " + e.getMessage());
            }
        }
        final Field nextCheckField = field;
        long nextCheck = System.currentTimeMillis();
        // the next part may throw exception, current time is the default value
        if (dailyRollingFileAppender != null && nextCheckField != null) {
            try {
                nextCheck = nextCheckField.getLong(dailyRollingFileAppender);
                scheduleWork(new DailyLogTask(),
                        nextCheck - System.currentTimeMillis() + 30 * 1000, 0, TimeUnit.MILLISECONDS);
            } catch (Exception e) {
                hostLog.error("Failed to set daily system info logging: " + e.getMessage());
            }
        }
    }