Processes messages received from kafka
331553
def _process_messages(self):
        """"""
        try:
            for message in self.consumer:
                try:
                    if message is None:
                        self.logger.debug("no message")
                        break
                    loaded_dict = json.loads(message.value)
                    self.logger.debug("got valid kafka message")

                    with self.uuids_lock:
                        if 'uuid' in loaded_dict:
                            if loaded_dict['uuid'] in self.uuids and \
                                    self.uuids[loaded_dict['uuid']] != 'poll':
                                self.logger.debug("Found Kafka message from request")
                                self.uuids[loaded_dict['uuid']] = loaded_dict
                            else:
                                self.logger.debug("Got poll result")
                                self._send_result_to_redis(loaded_dict)
                        else:
                            self.logger.debug("Got message not intended for this process")
                except ValueError:
                    extras = {}
                    if message is not None:
                            extras["data"] = message.value
                    self.logger.warning('Unparseable JSON Received from kafka',
                                                extra=extras)

            self._check_kafka_disconnect()

        except OffsetOutOfRangeError:
            # consumer has no idea where they are
            self.consumer.seek_to_end()
            self.logger.error("Kafka offset out of range error")
1055845
function onMessage(obj) {
    var id = obj.id;
    if (!self._matchesPrefix(id)) {
      debug('discarding unscoped message %s', id);
      return;
    }
    var cb = self._registry.get(id);
    if (cb) {
      process.nextTick(function () {
        debug('received message %s', id);
        // Ensure that the initial callback gets called asynchronously, even
        // for completely synchronous transports (otherwise the number of
        // pending requests will sometimes be inconsistent between stateful and
        // stateless transports).
        cb(null, Buffer.concat(obj.payload), self._adapter);
      });
    }
  }
829108
def ProcessMessage(self, message):
    """"""
    # Only accept authenticated messages
    if (message.auth_state !=
        rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):
      return

    now = time.time()

    # Maintain a cache of the foreman
    with self.lock:
      if (self.foreman_cache is None or
          now > self.foreman_cache.age + self.cache_refresh_time):
        self.foreman_cache = aff4.FACTORY.Open(
            "aff4:/foreman", mode="rw", token=self.token)
        self.foreman_cache.age = now

    if message.source:
      self.foreman_cache.AssignTasksToClient(message.source.Basename())
494157
def run(self):
        """"""
        log.debug("Starting Kafka producer I/O thread.")

        # main loop, runs until close is called
        while self._running:
            try:
                self.run_once()
            except Exception:
                log.exception("Uncaught error in kafka producer I/O thread")

        log.debug("Beginning shutdown of Kafka producer I/O thread, sending"
                  " remaining records.")

        # okay we stopped accepting requests but there may still be
        # requests in the accumulator or waiting for acknowledgment,
        # wait until these are completed.
        while (not self._force_close
               and (self._accumulator.has_unsent()
                    or self._client.in_flight_request_count() > 0)):
            try:
                self.run_once()
            except Exception:
                log.exception("Uncaught error in kafka producer I/O thread")

        if self._force_close:
            # We need to fail all the incomplete batches and wake up the
            # threads waiting on the futures.
            self._accumulator.abort_incomplete_batches()

        try:
            self._client.close()
        except Exception:
            log.exception("Failed to close network client")

        log.debug("Shutdown of Kafka producer I/O thread has completed.")
435948
private void _processInboxMessages(JSONArray messages) {
        synchronized (inboxControllerLock) {
            if (this.ctInboxController == null) {
                _initializeInbox();
            }
            if (this.ctInboxController != null) {
                boolean update = this.ctInboxController.updateMessages(messages);
                if (update) {
                    _notifyInboxMessagesDidUpdate();
                }
            }
        }
    }
1164654
def ReceiveMessagesRelationalFlows(self, client_id, messages):
    
    now = time.time()
    unprocessed_msgs = []
    message_handler_requests = []
    dropped_count = 0
    for session_id, msgs in iteritems(
        collection.Group(messages, operator.attrgetter("session_id"))):

      # Remove and handle messages to WellKnownFlows
      leftover_msgs = self.HandleWellKnownFlows(msgs)

      for msg in leftover_msgs:
        if (msg.auth_state != msg.AuthorizationState.AUTHENTICATED and
            msg.session_id != self.unauth_allowed_session_id):
          dropped_count += 1
          continue

        if session_id in queue_manager.session_id_map:
          message_handler_requests.append(
              rdf_objects.MessageHandlerRequest(
                  client_id=msg.source.Basename(),
                  handler_name=queue_manager.session_id_map[session_id],
                  request_id=msg.response_id,
                  request=msg.payload))
        else:
          unprocessed_msgs.append(msg)

    if dropped_count:
      logging.info("Dropped %d unauthenticated messages for %s", dropped_count,
                   client_id)

    if unprocessed_msgs:
      flow_responses = []
      for message in unprocessed_msgs:
        flow_responses.append(
            rdf_flow_objects.FlowResponseForLegacyResponse(message))

      data_store.REL_DB.WriteFlowResponses(flow_responses)

      for msg in unprocessed_msgs:
        if msg.type == rdf_flows.GrrMessage.Type.STATUS:
          stat = rdf_flows.GrrStatus(msg.payload)
          if stat.status == rdf_flows.GrrStatus.ReturnedStatus.CLIENT_KILLED:
            # A client crashed while performing an action, fire an event.
            crash_details = rdf_client.ClientCrash(
                client_id=client_id,
                session_id=msg.session_id,
                backtrace=stat.backtrace,
                crash_message=stat.error_message,
                nanny_status=stat.nanny_status,
                timestamp=rdfvalue.RDFDatetime.Now())
            events.Events.PublishEvent(
                "ClientCrash", crash_details, token=self.token)

    if message_handler_requests:
      data_store.REL_DB.WriteMessageHandlerRequests(message_handler_requests)

    logging.debug("Received %s messages from %s in %s sec", len(messages),
                  client_id,
                  time.time() - now)
550793
def consume_message(self, header, message):
        """"""
        logmessage = {
            "time": (time.time() % 1000) * 1000,
            "header": "",
            "message": message,
        }
        if header:
            logmessage["header"] = (
                json.dumps(header, indent=2) + "\n" + "----------------" + "\n"
            )
        if isinstance(message, dict):
            logmessage["message"] = (
                json.dumps(message, indent=2) + "\n" + "----------------" + "\n"
            )

        print("=== Consume ====\n{header}{message}".format(**logmessage))
        self.log.info("Received message @{time}".format(**logmessage))
        self.log.debug(
            "Received message @{time}\n{header}{message}".format(**logmessage)
        )
        time.sleep(0.1)
555529
def get_next_batch(self):
        
        messages = self.get_from_kafka()
        if messages:
            for message in messages:
                item = BaseRecord(message)
                self.increase_read()
                yield item

        self.logger.debug('Done reading batch')
        self.last_position = self.consumer.offsets
261544
protected void processMessagesOnConnection() {
        try {
            logger.log(INFO, "Connected to " + getConnectionName());
            state.set(StreamState.CONNECTED);
            notifyConnection();
            inputBuffer.flip();

            while (!Thread.currentThread().isInterrupted() && isConnected()) {
                try {
                    // Find the start of message
                    byte byStart = 0;
                    while(byStart != START_OF_MSG && !Thread.currentThread().isInterrupted() && isConnected()) {
                        byStart = nextByte(inputBuffer);
                    }

                    // and then make sure there are enough bytes and read the protocol
                    readCompleteMessage(inputBuffer);
                    if(inputBuffer.get() != protocol.getKeyIdentifier()) throw new TcProtocolException("Bad protocol");

                    logByteBuffer("Line read from stream", inputBuffer);

                    // now we take a shallow buffer copy and process the message
                    MenuCommand mc = protocol.fromChannel(inputBuffer);
                    logger.log(INFO, "Command received: " + mc);
                    notifyListeners(mc);
                }
                catch(TcProtocolException ex) {
                    // a protocol problem shouldn't drop the connection
                    logger.log(WARNING, "Probable Bad message reason='{0}' Remote={1} ",
                               ex.getMessage(), getConnectionName());
                }
            }
            logger.log(INFO, "Disconnected from " + getConnectionName());
        } catch (Exception e) {
            logger.log(ERROR, "Problem with connectivity on " + getConnectionName(), e);
        }
        finally {
            close();
        }
    }
786867
def process(self, senderKeyName, senderKeyDistributionMessage):
        
        senderKeyRecord = self.senderKeyStore.loadSenderKey(senderKeyName)
        senderKeyRecord.addSenderKeyState(senderKeyDistributionMessage.getId(),
                                          senderKeyDistributionMessage.getIteration(),
                                          senderKeyDistributionMessage.getChainKey(),
                                          senderKeyDistributionMessage.getSignatureKey())
        self.senderKeyStore.storeSenderKey(senderKeyName, senderKeyRecord)
1077473
def consume_messages(self, max_next_messages):
        
        # get messages list from kafka
        if self.__next_messages == 0:
            self.set_next_messages(min(1000, max_next_messages))
        self.set_next_messages(min(self.__next_messages, max_next_messages))
        mark = time.time()
        for record in self._get_messages_from_consumer():
            yield record.partition, record.offset, record.key, record.value
        newmark = time.time()
        if newmark - mark > 30:
            self.set_next_messages(self.__next_messages / 2 or 1)
        elif newmark - mark < 5:
            self.set_next_messages(min(self.__next_messages + 100, max_next_messages))
88996
def _process_msg(self, sid, subject, reply, data):
        
        payload_size = len(data)
        self.stats['in_msgs'] += 1
        self.stats['in_bytes'] += payload_size

        msg = Msg(subject=subject.decode(), reply=reply.decode(), data=data)

        # Don't process the message if the subscription has been removed
        sub = self._subs.get(sid)
        if sub is None:
            raise tornado.gen.Return()

        # Check if it is an old style request.
        if sub.future is not None:
            sub.future.set_result(msg)

            # Discard subscription since done
            self._subs.pop(sid, None)
            self._remove_subscription(sub)
            raise tornado.gen.Return()

        # Let subscription wait_for_msgs coroutine process the messages,
        # but in case sending to the subscription task would block,
        # then consider it to be an slow consumer and drop the message.
        try:
            sub.pending_size += payload_size
            if sub.pending_size >= sub.pending_bytes_limit:
                # Substract again the bytes since throwing away
                # the message so would not be pending data.
                sub.pending_size -= payload_size

                if self._error_cb is not None:
                    yield self._error_cb(ErrSlowConsumer())
                raise tornado.gen.Return()
            sub.pending_queue.put_nowait(msg)
        except tornado.queues.QueueFull:
            if self._error_cb is not None:
                yield self._error_cb(ErrSlowConsumer())
1314158
def ProcessResponses(self, responses, thread_pool):
    """"""
    for response in responses:
      thread_pool.AddTask(
          target=self._SafeProcessMessage,
          args=(response,),
          name=self.__class__.__name__)
365556
def process_message(self,msg,sock):
    ""
    command=msg[0]
    try: f={'GET':self.get,'SET':self.set,'SUBSCRIBE':self.sub,'PUBLISH':self.pub,
      'PING':self.ping,'GETSET':self.getset,'EXPIRE':self.expire,'DEL':self.delete}[command]
    except KeyError: print msg; raise
    args=msg[1:]
    try: return f(sock,*args) if command in self.SOCK_COMMANDS else f(*args)
    except Exception as e:
      print e
      print msg
      return '-ERROR\r\n'
1302592
def process_message(message)
      logger.info("Calling event #{message.message_type} in the channel #{message.channel_type}")
      @listener_channels[message.channel_type.to_sym].message(message.message_type.to_sym, message)
    end
464119
@Override
    public void processMessage() {
        if (overwrite)
            storage.setArray(key, payload);
        else if (!storage.arrayExists(key))
            storage.setArray(key, payload);
    }
292066
def process_events(args):
    """"""
    client = EventsClient.from_config()
    client.set_debug(True)
    if args.get:
        response = client.get_events()
    elif args.flush:
        response = client.flush_events()
    return response