Special handler for log messages.

        Accumulate incoming log messages and post them in batch.
493912
def process_log(self, **log_item):
        
        logger.debug("Processing log item: %s", log_item)
        self.log_batch.append(log_item)
        if len(self.log_batch) >= self.log_batch_size:
            self._post_log_batch()
303308
def add_log_handler(log, handler=None, debug=None, fmt=None):
    
    if debug:
        log.setLevel(debug)
    if handler:
        # if not fmt:
        #     fmt = __LOG_FMT
        if fmt:
            handler.setFormatter(fmt)
        log.addHandler(handler)
48371
def handle_collect(self, msg):
        """"""
        (success, sequence_number, comment) = self._handle_collect(msg)
        self.collector.send_multipart(msgs.MessageWriter().bool(success).uint64(sequence_number).string(comment).get())
848852
def add_messages(self, messages):
        
        # remove old messages
        if len(messages) > self.limit:
            messages = messages[:self.limit]
        _logger(self.__class__).log(5, 'adding %s messages' % len(messages))
        self._finished = False
        self._new_messages = messages
        self._to_check = list(self._messages)
        self._pending_msg = messages
        # start removing messages, new message won't be added until we
        # checked all message that need to be removed
        QtCore.QTimer.singleShot(1, self._remove_batch)
1177020
def _addToBuffers(self, logname, data):
        
        n = len(data)

        self.buflen += n
        self.buffered.append((logname, data))
        if self.buflen > self.BUFFER_SIZE:
            self._sendBuffers()
        elif not self.sendBuffersTimer:
            self.sendBuffersTimer = self._reactor.callLater(
                self.BUFFER_TIMEOUT, self._bufferTimeout)
424239
def on_receive(self, message):
        
        if self.log_file is not None and message['should_log']:
            message_copy = Message(message['text'],
                                   Metadata(None).__dict__,
                                   message['should_log']).__dict__
            with open(self.log_file, mode='w') as f:
                f.write(json.dumps(message_copy))
            logger.info(message['metadata']['source'])
        if self.listening_for(message):
            self_thread = self.HandlerThread(self.handle, message)
            self_thread.start()
        self.cleanup()
        for child in self.children:
            child.tell(message)
173114
private void cleanupLogs() throws IOException {
        logger.trace("Beginning log cleanup...");
        int total = 0;
        Iterator<Log> iter = getLogIterator();
        long startMs = System.currentTimeMillis();
        while (iter.hasNext()) {
            Log log = iter.next();
            total += cleanupExpiredSegments(log) + cleanupSegmentsToMaintainSize(log);
        }
        if (total > 0) {
            logger.warn("Log cleanup completed. " + total + " files deleted in " + (System.currentTimeMillis() - startMs) / 1000 + " seconds");
        } else {
            logger.trace("Log cleanup completed. " + total + " files deleted in " + (System.currentTimeMillis() - startMs) / 1000 + " seconds");
        }
    }
1023300
def emit(self, batch):
        
        try:
            self.client.submitBatches([batch])
            # it will call http_transport.flush() and
            # status code and message will be updated
            code = self.http_transport.code
            msg = self.http_transport.message
            if code >= 300 or code < 200:
                logging.error("Traces cannot be uploaded;\
                        HTTP status code: {}, message {}".format(code, msg))
        except Exception as e:  # pragma: NO COVER
            logging.error(getattr(e, 'message', e))

        finally:
            if self.http_transport.isOpen():
                self.http_transport.close()
1048824
def status_log(func, message, *args, **kwargs):
    """"""

    click.echo(message)
    log = func(*args, **kwargs)

    if log:
        out = []

        for line in log.split('\n'):
            if not line.startswith('#'):
                out.append(line)
        click.echo(black('\n'.join(out)))
353388
function log() {
	var msg = util.format.apply(null, Array.prototype.slice.call(arguments, 0));
	write(msg);
}
646759
def batch_message_from_parts(cls, messages):
        '''
        '''
        # Comma-separate the messages and wrap the lot in square brackets
        middle = b', '.join(messages)
        if not middle:
            raise ProtocolError.empty_batch()
        return b''.join([b'[', middle, b']'])
580070
private void doBatchMessage(final JsonObject message) {
    String batchID = message.getString("batch");
    if (currentBatch != null && currentBatch.id().equals(batchID)) {
      Object value = deserializer.deserialize(message);
      if (value != null) {
        if (log.isDebugEnabled()) {
          log.debug(String.format("%s - Batch received: Batch[batch=%s, id=%d, message=%s]", this, batchID, message.getLong("id"), value));
        }
        currentBatch.handleMessage(value);
      }
    }
  }
851216
def process_log_event(event, context):
    """"""
    init()
    serialized = event['awslogs'].pop('data')
    data = json.loads(zlib.decompress(
        base64.b64decode(serialized), 16 + zlib.MAX_WBITS))

    # Fetch additional logs for context (20s window)
    timestamps = [e['timestamp'] for e in data['logEvents']]
    start = min(timestamps) - 1000 * 15
    end = max(timestamps) + 1000 * 5

    events = logs.get_log_events(
        logGroupName=data['logGroup'],
        logStreamName=data['logStream'],
        startTime=start,
        endTime=end,
        startFromHead=True)['events']

    message = [
        "An error was detected",
        "",
        "Log Group: %s" % data['logGroup'],
        "Log Stream: %s" % data['logStream'],
        "Log Owner: %s" % data['owner'],
        "",
        "Log Contents",
        ""]

    # We may get things delivered from log sub that are not in log events
    for evt in data['logEvents']:
        if evt not in events:
            events.append(evt)

    for evt in events:
        message.append(message_event(evt))
        message.append("")

    params = dict(
        TopicArn=config['topic'],
        Subject=config['subject'],
        Message='\n'.join(message))

    sns.publish(**params)
982429
def do_handle_log(self, workunit, level, *msg_elements):
    """"""

    entry_info = {
      'level': self._log_level_str[level],
      'messages': self._render_messages(*msg_elements),
    }

    root_id = str(workunit.root().id)
    current_stack = self._root_id_to_workunit_stack[root_id]
    if current_stack:
      current_stack[-1]['log_entries'].append(entry_info)
    else:
      self.results[root_id]['log_entries'].append(entry_info)
1265815
def _sendBuffers(self):
        
        msg = {}
        msg_size = 0
        lastlog = None
        logdata = []
        while self.buffered:
            # Grab the next bits from the buffer
            logname, data = self.buffered.popleft()

            # If this log is different than the last one, then we have to send
            # out the message so far.  This is because the message is
            # transferred as a dictionary, which makes the ordering of keys
            # unspecified, and makes it impossible to interleave data from
            # different logs.  A future enhancement could be to change the
            # master to support a list of (logname, data) tuples instead of a
            # dictionary.
            # On our first pass through this loop lastlog is None
            if lastlog is None:
                lastlog = logname
            elif logname != lastlog:
                self._sendMessage(msg)
                msg = {}
                msg_size = 0
            lastlog = logname

            logdata = msg.setdefault(logname, [])

            # Chunkify the log data to make sure we're not sending more than
            # CHUNK_LIMIT at a time
            for chunk in self._chunkForSend(data):
                if not chunk:
                    continue
                logdata.append(chunk)
                msg_size += len(chunk)
                if msg_size >= self.CHUNK_LIMIT:
                    # We've gone beyond the chunk limit, so send out our
                    # message.  At worst this results in a message slightly
                    # larger than (2*CHUNK_LIMIT)-1
                    self._sendMessage(msg)
                    msg = {}
                    logdata = msg.setdefault(logname, [])
                    msg_size = 0
        self.buflen = 0
        if logdata:
            self._sendMessage(msg)
        if self.sendBuffersTimer:
            if self.sendBuffersTimer.active():
                self.sendBuffersTimer.cancel()
            self.sendBuffersTimer = None
605921
def log_progress(
    iterable,
    logger,
    batch_size=10000,
    progress_message="Processing... ({})",
    done_message="Done! (Total: {})",
):
    
    i = 0
    for x in iterable:
        yield x
        i += 1
        if not i % batch_size:
            logger.info(progress_message.format(i))
    logger.info(done_message.format(i))
112927
def log(message, level="info"):
    
    now = time.time()
    entry = {}
    entry['level'] = level
    entry['message'] = message
    entry['timestamp'] = now
    _log_entries.append(entry)