handling incoming log data
430967
def handle_log_data(self, m):
        ''''''
        if self.download_file is None:
            return
        # lose some data
        # import random
        # if random.uniform(0,1) < 0.05:
        #    print('dropping ', str(m))
        #    return
        if m.ofs != self.download_ofs:
            self.download_file.seek(m.ofs)
            self.download_ofs = m.ofs
        if m.count != 0:
            s = bytearray(m.data[:m.count])
            self.download_file.write(s)
            self.download_set.add(m.ofs // 90)
            self.download_ofs += m.count
        self.download_last_timestamp = time.time()
        if m.count == 0 or (m.count < 90 and len(self.download_set) == 1 + (m.ofs // 90)):
            dt = time.time() - self.download_start
            self.download_file.close()
            size = os.path.getsize(self.download_filename)
            speed = size / (1000.0 * dt)
            print("Finished downloading %s (%u bytes %u seconds, %.1f kbyte/sec %u retries)" % (
                self.download_filename,
                size,
                dt, speed,
                self.retries))
            self.download_file = None
            self.download_filename = None
            self.download_set = set()
            self.master.mav.log_request_end_send(self.target_system,
                                                 self.target_component)
            if len(self.download_queue):
                self.log_download_next()
35828
def handler(self,data):
		'''
		
		'''
		if isinstance(data,r.models.Response):
			self.log.debug("data is request object =  %s", str(data.content))
			data = data.content
		elif isinstance(data,str):
			self.log.info("data is json string with len %d",len(data))
			if len(data) == 0:
				self.log.warn("Handler received data of 0 length, exiting handler.")
				return
		else:
			self.log.error("Input is not valid request object or json string : %s" %str(data))
			return False
		try:
			data = json.loads(data)
			if 'async-responses' in data.keys():
				self.async_responses_callback(data)
			if 'notifications' in data.keys():
				self.notifications_callback(data)
			if 'registrations' in data.keys():
				self.registrations_callback(data)
			if 'reg-updates' in data.keys():
				self.reg_updates_callback(data)
			if 'de-registrations' in data.keys():
				self.de_registrations_callback(data)
			if 'registrations-expired' in data.keys():
				self.registrations_expired_callback(data)
		except:
			self.log.error("handle router had an issue and threw an exception")
			ex_type, ex, tb = sys.exc_info()
			traceback.print_tb(tb)
			self.log.error(sys.exc_info())
			del tb
726118
def log(fname,msg):
    '''  '''
    with open(fname,'a') as f:
        f.write(datetime.datetime.now().strftime('%m-%d-%Y %H:%M:\n') + msg + '\n')
1121945
def _handle_lines(self):
        """"""
        while b'\xdd' in self._buffer:
            linebuf, self._buffer = self._buffer.rsplit(b'\xdd', 1)
            line = linebuf[-19:]
            self._buffer += linebuf[:-19]
            if self._valid_packet(line):
                self._handle_raw_packet(line)
            else:
                self.logger.warning('dropping invalid data: %s',
                                    binascii.hexlify(line))
1223358
def parse_logs(self, f):
        """"""
        data = {}
        for l in f.splitlines():
            # catch empty lines
            if len(l) == 0:
                continue
            s = l.split("\t")
            data_ = []
            # catch lines with descriptive content: "Of pairs used:"
            for i in s[1:]:
                if len(i) == 0:
                    continue
                try:
                    i.replace('(', '')
                    i.replace(')', '')
                    i.replace(',', '')

                    data_.append(float(i))
                except ValueError:
                    data_.append(i)
            if len(data_) == 0:
                continue
            if s[0].startswith('short range'):
                s[0] = 'short range'
            elif s[0].startswith('same fragment'):
                s[0] = 'same fragment'
            s[0] = s[0].capitalize()
            data[s[0]] = data_
        return data
1023090
def do_handle_log(self, workunit, level, *msg_elements):
    """"""
    # If the element is a (msg, detail) pair, we ignore the detail. There's no
    # useful way to display it on the console.
    elements = [e if isinstance(e, string_types) else e[0] for e in msg_elements]
    msg = '\n' + ''.join(elements)
    if self.settings.color:
      msg = red(msg)
    self._emit(msg)
1174553
function ( callback ) {
		console.error( 'Getting %s logs since %s...', logType, start );

		client.getLog( logType, start, function ( err, data, next ) {
			const len = data && data.length;

			client.log( 'Got %s log entries', len );

			if ( len > 0 ) {
				// it's our first batch of data - format CSV header
				if ( start === '' ) {
					writeCsvLine( Object.keys( data[ 0 ] ) );
				}

				// print rows
				data.forEach( writeCsvLine );
			}

			// next time get next batch
			start = next;
			callback( next ? null : 'no more data' );
		} );
	}
424239
def on_receive(self, message):
        
        if self.log_file is not None and message['should_log']:
            message_copy = Message(message['text'],
                                   Metadata(None).__dict__,
                                   message['should_log']).__dict__
            with open(self.log_file, mode='w') as f:
                f.write(json.dumps(message_copy))
            logger.info(message['metadata']['source'])
        if self.listening_for(message):
            self_thread = self.HandlerThread(self.handle, message)
            self_thread.start()
        self.cleanup()
        for child in self.children:
            child.tell(message)
975009
def pipe_data_received(self, fd, data):
        """"""
        if fd == 2:  # stderr fd number
            self._on_stderr(data)
        elif self._on_data:
            self._on_data(data)
        else:
            self._queued_data.append(data)
47471
def data_received(self, data):
        """"""
        # A message might be split over several reads, so we store a buffer and
        # try to decode messages from that buffer
        self._buffer += data
        log_binary(_LOGGER, '<< Receive', Data=data)

        while self._buffer:
            # The variant tells us how much data must follow
            length, raw = read_variant(self._buffer)
            if len(raw) < length:
                _LOGGER.debug(
                    'Require %d bytes but only %d in buffer', length, len(raw))
                break

            data = raw[:length]  # Incoming message (might be encrypted)
            self._buffer = raw[length:]  # Buffer, might contain more messages

            try:
                self._handle_message(data)
            except Exception:  # pylint: disable=broad-except
                _LOGGER.error('Failed to handle message')
995123
def _log_request(request):
    """"""
    logger.debug("Inbound email received")

    for k, v in list(request.POST.items()):
        logger.debug("- POST['%s']='%s'" % (k, v))

    for n, f in list(request.FILES.items()):
        logger.debug("- FILES['%s']: '%s', %sB", n, f.content_type, f.size)
336929
def data_received(self, data):
        """"""
        self._data_buffer += data.decode()
        if not self._data_buffer.endswith('\r\n'):
            return
        data = self._data_buffer
        self._data_buffer = ''  # clear buffer
        for cmd in data.strip().split('\r\n'):
            data = json.loads(cmd)
            if not isinstance(data, list):
                data = [data]
            for item in data:
                self.handle_data(item)
247744
private void handleLogfile() {
        if (args.logFile != null) {
            FileUtils.close(out);
            FileUtils.close(err);
        }
    }
1242590
def log_url (self, url_data):
        """"""
        self.check_active_loggers()
        do_print = self.do_print(url_data)
        # Only send a transport object to the loggers, not the complete
        # object instance.
        for log in self.loggers:
            log.log_filter_url(url_data, do_print)
407964
def logs(context, format):
    """"""
    fritz = context.obj
    fritz.login()

    messages = fritz.get_logs()
    if format == "plain":
        for msg in messages:
            merged = "{} {} {}".format(msg.date, msg.time, msg.message.encode("UTF-8"))
            click.echo(merged)

    if format == "json":
        entries = [msg._asdict() for msg in messages]
        click.echo(json.dumps({
            "entries": entries,
        }))
325094
def log(self, msg):
        '''
        '''
        if self.logfile:
            fd = self.logfile.fileno()
        else:
            fd = sys.stdout.fileno()

        os.write(fd, ('%.3f %s\n' % (time.time(), msg)).encode('UTF-8'))
458689
def parse_log(log, url_pattern):
    
    hits = {}
    initial_timestamp = None

    def clean_timestamp(v):
        return ' '.join(v).strip('[]')
    for line in log:
        data = line.split()
        if not initial_timestamp:
            initial_timestamp = clean_timestamp(data[3:5])
        match = url_pattern.match(data[6])
        if match:
            ident_hash = '@'.join(match.groups())
            if ident_hash:
                hits[ident_hash] = hits.get(ident_hash, 0) + 1
    else:
        end_timestamp = clean_timestamp(data[3:5])
    return hits, initial_timestamp, end_timestamp