process incoming messages

        data is a dict with several arrays
545648
def process_incoming(self):
        

        model = self.model
        sockets = self.sockets

        # Check for new messages
        if not sockets:
            return
        # unpack sockets
        poller = sockets['poller']
        rep = sockets['rep']
        # pull = sockets['pull']
        pub = sockets['pub']
        items = poller.poll(10)
        for sock, n in items:
            for i in range(n):

                A, metadata = recv_array(sock)
                var = None

                # bmi actions
                if "update" in metadata:
                    dt = float(metadata["update"])
                    model.update(dt)
                    metadata["dt"] = dt
                elif "get_var" in metadata:
                    name = metadata["get_var"]
                    # temporary implementation
                    if metadata.get("copy", False):
                        var = model.get_var(name).copy()
                    else:
                        var = model.get_var(name)
                    if var is None:
                        logger.warning("Get_var returns None for %s" % name)
                    metadata['name'] = name
                    # assert socket is req socket
                elif "get_var_count" in metadata:
                    # temporary implementation
                    n = model.get_var_count()
                    metadata['get_var_count'] = n
                    # assert socket is req socket
                elif "get_var_rank" in metadata:
                    # temporary implementation
                    var_name = metadata['get_var_rank']
                    n = model.get_var_rank(var_name)
                    metadata['get_var_rank'] = n
                    # assert socket is req socket
                elif "get_var_shape" in metadata:
                    # temporary implementation
                    var_name = metadata['get_var_shape']
                    n = model.get_var_shape(var_name)
                    metadata['get_var_shape'] = tuple([int(item) for item in n])
                    # assert socket is req socket
                elif "get_var_type" in metadata:
                    # temporary implementation
                    var_name = metadata['get_var_type']
                    n = model.get_var_type(var_name)
                    metadata['get_var_type'] = n
                    # assert socket is req socket
                elif "get_var_name" in metadata:
                    i = int(metadata["get_var_name"])
                    name = model.get_var_name(i)
                    metadata['get_var_name'] = name
                    # assert socket is req socket
                elif "set_current_time" in metadata:
                    t = float(metadata["set_current_time"])
                    model.set_current_time(t)
                elif "set_var" in metadata:
                    name = metadata["set_var"]
                    # logger.debug("setting variable %s", name)
                    model.set_var(name, A)
                    metadata["name"] = name  # !?
                elif "set_var_slice" in metadata:
                    name = metadata["set_var_slice"]
                    logger.debug("setting variable %s", name)
                    start = metadata["start"]
                    count = metadata["count"]
                    model.set_var_slice(name, start, count, A)
                    metadata["name"] = name  # !?
                elif "set_var_index" in metadata:
                    name = metadata["set_var_index"]
                    logger.debug("setting variable %s using index index", name)
                    index = metadata["index"]
                    # TODO: test if this is fast enough.
                    # Otherwise move to BMI ++ but that is
                    # a bit of a burden on implementers
                    var = model.get_var(name).copy()
                    var.flat[index] = A
                    model.set_var(name, var)
                    metadata["name"] = name  # !?
                elif "get_current_time" in metadata:
                    metadata["get_current_time"]
                    t = model.get_current_time()
                    metadata['get_current_time'] = t
                elif "get_time_step" in metadata:
                    metadata["get_time_step"]
                    dt = model.get_time_step()
                    metadata['get_time_step'] = dt
                elif "get_end_time" in metadata:
                    metadata["get_end_time"]
                    t = model.get_end_time()
                    metadata['get_end_time'] = t
                elif "get_start_time" in metadata:
                    metadata["get_start_time"]
                    t = model.get_start_time()
                    metadata['get_start_time'] = t
                    # assert socket is req socket
                # custom actions
                elif "remote" in metadata:
                    assert metadata["remote"] in {
                        "play", "stop", "pause", "rewind", "quit"}
                    model.state = metadata["remote"]
                elif "operator" in metadata:
                    # TODO: support same operators as MPI_ops here....,
                    # TODO: reduce before apply
                    # TODO: assert pull socket
                    pass
                    # S = tuple(slice(*x) for x in action['slice'])
                    # print(repr(arr[S]))
                    # if action['operator'] == 'setitem':
                    #     arr[S] = data
                    # elif action['operator'] == 'add':
                    #     arr[S] += data
                elif "initialize" in metadata:
                    config_file = metadata["initialize"]
                    model.initialize(str(config_file))
                elif "finalize" in metadata:
                    model.finalize()
                else:
                    logger.warn("got unknown message {} from socket {}".format(str(metadata), sock))
                if sock.socket_type == zmq.REP:
                    # reply
                    send_array(rep, var, metadata=metadata)
                # any getter requested through the pull socket?
                elif any(x.startswith("get_") for x in metadata) and sock.socket_type == zmq.PULL:
                    # return through the pub socket
                    send_array(pub, var, metadata=metadata)
620559
def parse(data):
    

    # convert to string
    data = data.decode("ascii")

    if len(data) == 2 and data == "A5":
        return AckMessage()

    # split into bytes
    raw = [data[i:i+2] for i in range(len(data)) if i % 2 == 0]

    if len(raw) != 7:
        return UnknownMessage(raw)

    if raw[1] == "B8":
        return StateMessage(raw)
    elif raw[3] == "12":
        return CommandMessage(raw)
    elif raw[3] == "14":
        return ScenarioTriggeredMessage(raw)
    elif raw[3] == "15":
        return RequestStatusMessage(raw)
    else:
        return UnknownMessage(raw)
331553
def _process_messages(self):
        """"""
        try:
            for message in self.consumer:
                try:
                    if message is None:
                        self.logger.debug("no message")
                        break
                    loaded_dict = json.loads(message.value)
                    self.logger.debug("got valid kafka message")

                    with self.uuids_lock:
                        if 'uuid' in loaded_dict:
                            if loaded_dict['uuid'] in self.uuids and \
                                    self.uuids[loaded_dict['uuid']] != 'poll':
                                self.logger.debug("Found Kafka message from request")
                                self.uuids[loaded_dict['uuid']] = loaded_dict
                            else:
                                self.logger.debug("Got poll result")
                                self._send_result_to_redis(loaded_dict)
                        else:
                            self.logger.debug("Got message not intended for this process")
                except ValueError:
                    extras = {}
                    if message is not None:
                            extras["data"] = message.value
                    self.logger.warning('Unparseable JSON Received from kafka',
                                                extra=extras)

            self._check_kafka_disconnect()

        except OffsetOutOfRangeError:
            # consumer has no idea where they are
            self.consumer.seek_to_end()
            self.logger.error("Kafka offset out of range error")
1211570
def deserialize_email_messages(messages: List[str]):
    """"""
    return [
        pickle.loads(zlib.decompress(base64.b64decode(m)))
        for m in messages
    ]
250640
function parse(data, initExpected) {
		var lines, i, last;
		data = tail + data;
		data = data.replace(/\r*\n+/g, "\n");
		lines = data.split("\n");
		last = lines.length - 1;
		for (i = 0; i < last; i++) {
			messages.push(read(lines[i], initExpected));
		}
		tail = lines[lines.length - 1];
	}
336929
def data_received(self, data):
        """"""
        self._data_buffer += data.decode()
        if not self._data_buffer.endswith('\r\n'):
            return
        data = self._data_buffer
        self._data_buffer = ''  # clear buffer
        for cmd in data.strip().split('\r\n'):
            data = json.loads(cmd)
            if not isinstance(data, list):
                data = [data]
            for item in data:
                self.handle_data(item)
38762
def process_msg(self, msg):
        """"""
        jmsg = json.loads(msg)
        msgtype = jmsg['MessageType']
        msgdata = jmsg['Data']

        _LOGGER.debug('New websocket message recieved of type: %s', msgtype)
        if msgtype == 'Sessions':
            self._sessions = msgdata
            # Check for new devices and update as needed.
            self.update_device_list(self._sessions)
        """
        May process other message types in the future.
        Other known types are:
        - PlaybackStarted
        - PlaybackStopped
        - SessionEnded
        """
513357
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'message_type' in _dict:
            args['message_type'] = _dict.get('message_type')
        if 'text' in _dict:
            args['text'] = _dict.get('text')
        if 'options' in _dict:
            args['options'] = MessageInputOptions._from_dict(
                _dict.get('options'))
        if 'intents' in _dict:
            args['intents'] = [
                RuntimeIntent._from_dict(x) for x in (_dict.get('intents'))
            ]
        if 'entities' in _dict:
            args['entities'] = [
                RuntimeEntity._from_dict(x) for x in (_dict.get('entities'))
            ]
        if 'suggestion_id' in _dict:
            args['suggestion_id'] = _dict.get('suggestion_id')
        return cls(**args)
958861
def from_array(array):
        
        if array is None or not array:
            return None
        # end if
        assert_type_or_raise(array, dict, parameter_name="array")
        from pytgbot.api_types.receivable.peer import User

        data = {}
        data['id'] = u(array.get('id'))
        data['from_peer'] = User.from_array(array.get('from'))
        data['invoice_payload'] = u(array.get('invoice_payload'))
        data['shipping_address'] = ShippingAddress.from_array(array.get('shipping_address'))
        data['_raw'] = array
        return ShippingQuery(**data)
826389
def process_array_items(self, array, json):
        
        for item in json['items']:
            key = None
            processed = self.from_json(item)

            if isinstance(processed, Asset):
                key = 'Asset'
            elif isinstance(processed, Entry):
                key = 'Entry'

            if key is not None:
                array.items_mapped[key][processed.sys['id']] = processed

            array.items.append(processed)
848505
async def message_data(request):
    
    messages = []
    # {% if database.is_none %}
    if request.app['settings'].MESSAGE_FILE.exists():
        # read the message file, process it and populate the "messages" list
        with request.app['settings'].MESSAGE_FILE.open() as msg_file:
            for line in msg_file:
                if not line:
                    # ignore blank lines eg. end of file
                    continue
                # split the line into it constituent parts, see process_form above
                username, ts, message = line.split('|', 2)
                # parse the datetime string and render it in a more readable format.
                ts = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.strptime(ts, '%Y-%m-%dT%H:%M:%S.%f'))
                messages.append({'username': username, 'timestamp': ts, 'message': message})
        messages.reverse()
    # {% elif database.is_pg_sqlalchemy %}

    async with request.app['pg_engine'].acquire() as conn:
        async for row in conn.execute(sa_messages.select().order_by(sa_messages.c.timestamp.desc())):
            ts = '{:%Y-%m-%d %H:%M:%S}'.format(row.timestamp)
            messages.append({'username': row.username, 'timestamp': ts, 'message': row.message})
    # {% endif %}
    return json_response(messages)
260707
def Process(self, fs_msg, context):
    """"""
    try:
      if fs_msg.message_type == "GrrMessage":
        grr_message = rdf_flows.GrrMessage.FromSerializedString(
            fs_msg.data.value)
        self._ProcessGRRMessages(fs_msg.source.client_id, [grr_message])
      elif fs_msg.message_type == "MessageList":
        packed_messages = rdf_flows.PackedMessageList.FromSerializedString(
            fs_msg.data.value)
        message_list = communicator.Communicator.DecompressMessageList(
            packed_messages)
        self._ProcessGRRMessages(fs_msg.source.client_id, message_list.job)
      else:
        logging.error("Received message with unrecognized message_type: %s",
                      fs_msg.message_type)
        context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    except Exception as e:
      logging.error("Exception processing message: %s", str(e))
      raise
365556
def process_message(self,msg,sock):
    ""
    command=msg[0]
    try: f={'GET':self.get,'SET':self.set,'SUBSCRIBE':self.sub,'PUBLISH':self.pub,
      'PING':self.ping,'GETSET':self.getset,'EXPIRE':self.expire,'DEL':self.delete}[command]
    except KeyError: print msg; raise
    args=msg[1:]
    try: return f(sock,*args) if command in self.SOCK_COMMANDS else f(*args)
    except Exception as e:
      print e
      print msg
      return '-ERROR\r\n'
252686
def from_array(array):
        
        if array is None or not array:
            return None
        # end if
        assert_type_or_raise(array, dict, parameter_name="array")
        from pytgbot.api_types.receivable.payments import ShippingAddress
        

        data = {}
        data['name'] = u(array.get('name')) if array.get('name') is not None else None
        data['phone_number'] = u(array.get('phone_number')) if array.get('phone_number') is not None else None
        data['email'] = u(array.get('email')) if array.get('email') is not None else None
        data['shipping_address'] = ShippingAddress.from_array(array.get('shipping_address')) if array.get('shipping_address') is not None else None
        data['_raw'] = array
        return OrderInfo(**data)
954498
def from_array(array):
        
        if array is None or not array:
            return None
        # end if
        assert_type_or_raise(array, dict, parameter_name="array")
        from pytgbot.api_types.receivable.media import ChatPhoto
        from pytgbot.api_types.receivable.updates import Message
        

        data = {}
        data['id'] = int(array.get('id'))
        data['type'] = u(array.get('type'))
        data['title'] = u(array.get('title')) if array.get('title') is not None else None
        data['username'] = u(array.get('username')) if array.get('username') is not None else None
        data['first_name'] = u(array.get('first_name')) if array.get('first_name') is not None else None
        data['last_name'] = u(array.get('last_name')) if array.get('last_name') is not None else None
        data['all_members_are_administrators'] = bool(array.get('all_members_are_administrators')) if array.get('all_members_are_administrators') is not None else None
        data['photo'] = ChatPhoto.from_array(array.get('photo')) if array.get('photo') is not None else None
        data['description'] = u(array.get('description')) if array.get('description') is not None else None
        data['invite_link'] = u(array.get('invite_link')) if array.get('invite_link') is not None else None
        data['pinned_message'] = Message.from_array(array.get('pinned_message')) if array.get('pinned_message') is not None else None
        data['sticker_set_name'] = u(array.get('sticker_set_name')) if array.get('sticker_set_name') is not None else None
        data['can_set_sticker_set'] = bool(array.get('can_set_sticker_set')) if array.get('can_set_sticker_set') is not None else None
        data['_raw'] = array
        return Chat(**data)
633150
def data_received(self, data):
        
        _LOGGER.debug("Starting: data_received")
        _LOGGER.debug('Received %d bytes from PLM: %s',
                      len(data), binascii.hexlify(data))
        self._buffer.put_nowait(data)
        asyncio.ensure_future(self._peel_messages_from_buffer(),
                              loop=self._loop)

        _LOGGER.debug("Finishing: data_received")
985355
def _process_json(data):
    
    requests = []
    for item in data:
        committee = GradCommittee()
        committee.status = item.get('status')
        committee.committee_type = item.get('committeeType')
        committee.dept = item.get('dept')
        committee.degree_title = item.get('degreeTitle')
        committee.degree_type = item.get('degreeType')
        committee.major_full_name = item.get('majorFullName')
        committee.start_date = datetime_from_string(item.get('startDate'))
        committee.end_date = datetime_from_string(item.get('endDate'))
        for member in item.get('members'):
            if member.get('status') == "inactive":
                continue

            com_mem = GradCommitteeMember()
            com_mem.first_name = member.get('nameFirst')
            com_mem.last_name = member.get('nameLast')

            if member.get('memberType') is not None and\
                    len(member.get('memberType')) > 0:
                com_mem.member_type = member.get('memberType').lower()

            if member.get('readingType') is not None and\
                    len(member.get('readingType')) > 0:
                com_mem.reading_type = member.get('readingType').lower()

            com_mem.dept = member.get('dept')
            com_mem.email = member.get('email')
            com_mem.status = member.get('status')
            committee.members.append(com_mem)

        requests.append(committee)
    return requests