perform_work

        Process handler function for processing messages
        found in the ``multiprocessing.Manager.queue``

        Build the ``self.log_payload`` from the queued log messages
        and POST it to the Splunk endpoint
195112
def perform_work(
            self):
        

        self.debug_log((
            'perform_work - start'))

        self.tid = threading.current_thread().ident

        if self.sleep_interval > 0:
            # Stop the timer. Happens automatically if this is called
            # via the timer, does not if invoked by force_flush()
            if self.timer:
                self.timer.cancel()
            else:
                self.debug_log((
                    'perform_work - done - timer was destroyed'))
                return

        self.debug_log((
            'perform_work - ready tid={}').format(
                self.tid))

        try:

            not_done = not self.is_shutting_down(
                shutdown_event=self.shutdown_event)

            while not_done:

                try:
                    self.build_payload_from_queued_messages(
                        use_queue=self.queue,
                        shutdown_event=self.shutdown_event)
                except Exception as e:
                    if self.is_shutting_down(
                            shutdown_event=self.shutdown_event):
                        self.debug_log(
                            'perform_work - done - detected shutdown')
                    else:
                        self.debug_log((
                            'perform_work - done - '
                            'Exception shutting down with ex={}').format(
                                e))
                        self.shutdown()
                    self.already_done.set()
                    return
                # end of try to return if the queue

                self.publish_to_splunk()
                timer_interval = self.sleep_interval

                if not_done:
                    if self.is_shutting_down(
                            shutdown_event=self.shutdown_event):
                        self.debug_log(
                            'perform_work - done - shutdown detected')
                        self.already_done.set()
                        return
                    # check if done
                    else:
                        if self.num_sent > 0:
                            # run again if queue was not cleared
                            timer_interval = 0.5
                            self.debug_log(
                                'queue not empty - run again')
                        else:
                            self.debug_log(
                                'sleep={}s'.format(
                                    timer_interval))
                        time.sleep(timer_interval)
                # end of checking if this should run again or is shutting down

            # end of while not done

        except Exception as e:
            if self.is_shutting_down(
                    shutdown_event=self.shutdown_event):
                self.debug_log((
                    'perform_work - shutdown sent={}').format(
                        self.num_sent))
            else:
                self.debug_log((
                    'perform_work - ex={}').format(
                        e))
        # end of try/ex

        self.already_done.set()

        self.debug_log((
            'EXIT - perform_work - done - tid={}').format(
                self.tid))
1304203
def _schedule_log_parsing(job, job_logs, result):
    

    # importing here to avoid an import loop
    from treeherder.log_parser.tasks import parse_logs

    task_types = {
        "errorsummary_json",
        "buildbot_text",
        "builds-4h"
    }

    job_log_ids = []
    for job_log in job_logs:
        # a log can be submitted already parsed.  So only schedule
        # a parsing task if it's ``pending``
        # the submitter is then responsible for submitting the
        # text_log_summary artifact
        if job_log.status != JobLog.PENDING:
            continue

        # if this is not a known type of log, abort parse
        if job_log.name not in task_types:
            continue

        job_log_ids.append(job_log.id)

    # TODO: Replace the use of different queues for failures vs not with the
    # RabbitMQ priority feature (since the idea behind separate queues was
    # only to ensure failures are dealt with first if there is a backlog).
    if result != 'success':
        queue = 'log_parser_fail'
        priority = 'failures'
    else:
        queue = 'log_parser'
        priority = "normal"

    parse_logs.apply_async(queue=queue,
                           args=[job.id, job_log_ids, priority])
231693
def process_messages_loop(self):
        
        self.receiving_messages = True
        try:
            self.process_messages_loop_internal()
        except pika.exceptions.ConnectionClosed as ex:
            logging.error("Connection closed {}.".format(ex))
            raise
493912
def process_log(self, **log_item):
        
        logger.debug("Processing log item: %s", log_item)
        self.log_batch.append(log_item)
        if len(self.log_batch) >= self.log_batch_size:
            self._post_log_batch()
1295374
def handle_processing_packets():
    

    host = os.getenv(
        "LISTEN_ON_HOST",
        "127.0.0.1").strip().lstrip()
    port = int(os.getenv(
        "LISTEN_ON_PORT",
        "80").strip().lstrip())
    backlog = int(os.getenv(
        "LISTEN_BACKLOG",
        "5").strip().lstrip())
    size = int(os.getenv(
        "LISTEN_SIZE",
        "102400").strip().lstrip())
    sleep_in_seconds = float(os.getenv(
        "LISTEN_SLEEP",
        "0.5").strip().lstrip())
    needs_response = bool(os.getenv(
        "LISTEN_SEND_RESPONSE",
        "0").strip().lstrip() == "1")
    shutdown_hook = os.getenv(
        "LISTEN_SHUTDOWN_HOOK",
        "/tmp/shutdown-listen-server-{}-{}".format(
            host,
            port)).strip().lstrip()
    filter_key = os.getenv(
        "IGNORE_KEY",
        INCLUDED_IGNORE_KEY).strip().lstrip()

    if os.path.exists(shutdown_hook):
        log.info(("Please remove the shutdown hook file: "
                  "\nrm -f {}")
                 .format(
                    shutdown_hook))
        sys.exit(1)

    default_filter_key = filter_key
    bytes_for_filter_key = len(default_filter_key)
    offset_to_filter_key = (-1 * bytes_for_filter_key)
    offset_to_msg = offset_to_filter_key - 1

    now = datetime.datetime.now().isoformat()
    log.info(("{} - Starting Server address={}:{} "
              "backlog={} size={} sleep={} shutdown={} "
              "filter_key={}")
             .format(
                now,
                host,
                port,
                backlog,
                size,
                sleep_in_seconds,
                shutdown_hook,
                default_filter_key))

    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((host, port))
    s.listen(backlog)
    client, address = s.accept()

    midx = 0
    while 1:
        data = None
        address = None
        ignore_key = None

        try:
            if not client:
                client, address = s.accept()
        except Exception as e:
            log.error(("socket accept with ex={}")
                      .format(
                        e))
        try:
            if client:
                data = client.recv(size)
        except Exception as e:
            log.error(("recv - disconnected with ex={}")
                      .format(
                        e))
        if data:
            now = datetime.datetime.now().isoformat()
            packet_to_process = data[0:offset_to_msg]
            ignore_key = data[offset_to_filter_key:]
            log.info(("decoding data={} key={}")
                     .format(
                        packet_to_process,
                        ignore_key))

            msg = None
            try:
                msg = json.loads(
                            packet_to_process.decode("utf-8"))
            except Exception as e:
                msg = None
                log.error(("Invalid data={} with ex={}")
                          .format(
                            packet_to_process,
                            e))

            if msg:

                log.info(("received msg={} "
                          "data={} replying - ignore='{}'")
                         .format(
                            ppj(msg),
                            packet_to_process,
                            ignore_key))

                if msg["status"] == VALID:
                    if msg["data_type"] == TCP:
                        log.info("TCP")
                    elif msg["data_type"] == UDP:
                        log.info("TCP")
                    elif msg["data_type"] == ARP:
                        log.info("TCP")
                    elif msg["data_type"] == ICMP:
                        log.info("TCP")
                    else:
                        log.error(("unsuppported type={}")
                                  .format(
                                    msg["data_type"]))
                    # end of supported eth protocol message types
                else:
                    log.error(("unsuppported msg status={}")
                              .format(
                                msg["status"]))
                # end if msg was VALID
            # end of if found msg

            midx += 1
            if midx > 1000000:
                midx = 0
        else:
            log.debug("ignoring invalid data")
        # end of if valid msg or not

        if needs_response:
            client.send(ignore_key)
        else:
            log.info("no response")
            time.sleep(sleep_in_seconds)

        if os.path.exists(shutdown_hook):
            now = datetime.datetime.now().isoformat()
            log.info(("{} detected shutdown "
                      "file={}")
                     .format(
                        now,
                        shutdown_hook))
    # end of loop

    log.info("shutting down")
    client.close()

    log.info("done")
46748
async def run(self):
        
        # Implement pull max batch from queue instead of one message at a time
        while self.eh_partition_pump.pump_status != "Errored" and not self.eh_partition_pump.is_closing():
            if self.eh_partition_pump.partition_receive_handler:
                try:
                    msgs = await self.eh_partition_pump.partition_receive_handler.receive(
                        max_batch_size=self.max_batch_size,
                        timeout=self.recieve_timeout)
                except Exception as e:  # pylint: disable=broad-except
                    _logger.info("Error raised while attempting to receive messages: %r", e)
                    await self.process_error_async(e)
                else:
                    if not msgs:
                        _logger.info("No events received, queue size %r, release %r",
                                     self.eh_partition_pump.partition_receive_handler.queue_size,
                                     self.eh_partition_pump.host.eph_options.release_pump_on_timeout)
                        if self.eh_partition_pump.host.eph_options.release_pump_on_timeout:
                            await self.process_error_async(TimeoutError("No events received"))
                    else:
                        await self.process_events_async(msgs)
1358540
def process_queue
      begin
        while blk = Merb::Dispatcher.work_queue.pop
           # we've been blocking on the queue waiting for an item sleeping.
           # when someone pushes an item it wakes up this thread so we 
           # immediately pass execution to the scheduler so we don't 
           # accidentally run this block before the action finishes 
           # it's own processing
          Thread.pass
          blk.call
          break if Merb::Dispatcher.work_queue.empty? && Merb.exiting
        end
      rescue Exception => e
        Merb.logger.warn! %Q!Worker Thread Crashed with Exception:\n#{Merb.exception(e)}\nRestarting Worker Thread!
        retry
      end
    end
424239
def on_receive(self, message):
        
        if self.log_file is not None and message['should_log']:
            message_copy = Message(message['text'],
                                   Metadata(None).__dict__,
                                   message['should_log']).__dict__
            with open(self.log_file, mode='w') as f:
                f.write(json.dumps(message_copy))
            logger.info(message['metadata']['source'])
        if self.listening_for(message):
            self_thread = self.HandlerThread(self.handle, message)
            self_thread.start()
        self.cleanup()
        for child in self.children:
            child.tell(message)
695237
def mount_process(self, process):
    

    for route_path in process.route_paths:
      route = '/%s%s' % (process.pid.id, route_path)
      log.info('Mounting route %s' % route)
      self.app.add_handlers('.*$', [(
          re.escape(route),
          RoutedRequestHandler,
          dict(process=process, path=route_path)
      )])

    for message_name in process.message_names:
      route = '/%s/%s' % (process.pid.id, message_name)
      log.info('Mounting message handler %s' % route)
      self.app.add_handlers('.*$', [(
          re.escape(route),
          WireProtocolMessageHandler,
          dict(process=process, name=message_name)
      )])
31288
def build_into_splunk_tcp_message(
            self,
            body):
        
        # is for an authenticated Splunk TCP Port
        log_msg = None
        if self.token:
            if IS_PY2:
                log_msg = ('token={}, body={}').format(
                    self.token,
                    body)
            else:
                log_msg = ('token={}, body={}').format(
                    self.token,
                    body).encode()
        else:
            if IS_PY2:
                log_msg = ('{}').format(
                    body)
            else:
                log_msg = body.encode()
        # end of support for building different
        # Splunk socket-ready messages

        self.debug_log('build={}'.format(
            log_msg))

        return log_msg
446111
function worker(message) {
    console.log(`Processing "${message.message.data}"...`);

    setTimeout(() => {
      console.log(`Finished procesing "${message.message.data}".`);
      isProcessed = true;
    }, 30000);
  }
1339186
def handle_decoded_payload(self, data):
    '''
    
    '''
    # Ensure payload is unicode. Disregard failure to decode binary blobs.
    if six.PY2:
        data = salt.utils.data.decode(data, keep=True)
    if 'user' in data:
        log.info(
            'User %s Executing command %s with jid %s',
            data['user'], data['fun'], data['jid']
        )
    else:
        log.info(
            'Executing command %s with jid %s',
            data['fun'], data['jid']
        )
    log.debug('Command details %s', data)

    # Don't duplicate jobs
    log.trace('Started JIDs: %s', self.jid_queue)
    if self.jid_queue is not None:
        if data['jid'] in self.jid_queue:
            return
        else:
            self.jid_queue.append(data['jid'])
            if len(self.jid_queue) > self.opts['minion_jid_queue_hwm']:
                self.jid_queue.pop(0)

    if isinstance(data['fun'], six.string_types):
        if data['fun'] == 'sys.reload_modules':
            self.functions, self.returners, self.function_errors, self.executors = self._load_modules()
            self.schedule.functions = self.functions
            self.schedule.returners = self.returners

    process_count_max = self.opts.get('process_count_max')
    process_count_max_sleep_secs = self.opts.get('process_count_max_sleep_secs')
    if process_count_max > 0:
        process_count = len(salt.utils.minion.running(self.opts))
        while process_count >= process_count_max:
            log.warning('Maximum number of processes (%s) reached while '
                        'executing jid %s, waiting %s seconds...',
                        process_count_max,
                        data['jid'],
                        process_count_max_sleep_secs)
            yield tornado.gen.sleep(process_count_max_sleep_secs)
            process_count = len(salt.utils.minion.running(self.opts))

    # We stash an instance references to allow for the socket
    # communication in Windows. You can't pickle functions, and thus
    # python needs to be able to reconstruct the reference on the other
    # side.
    instance = self
    multiprocessing_enabled = self.opts.get('multiprocessing', True)
    if multiprocessing_enabled:
        if sys.platform.startswith('win'):
            # let python reconstruct the minion on the other side if we're
            # running on windows
            instance = None
        with default_signals(signal.SIGINT, signal.SIGTERM):
            process = SignalHandlingMultiprocessingProcess(
                target=self._target, args=(instance, self.opts, data, self.connected)
            )
    else:
        process = threading.Thread(
            target=self._target,
            args=(instance, self.opts, data, self.connected),
            name=data['jid']
        )

    if multiprocessing_enabled:
        with default_signals(signal.SIGINT, signal.SIGTERM):
            # Reset current signals before starting the process in
            # order not to inherit the current signal handlers
            process.start()
    else:
        process.start()

    # TODO: remove the windows specific check?
    if multiprocessing_enabled and not salt.utils.platform.is_windows():
        # we only want to join() immediately if we are daemonizing a process
        process.join()
    else:
        self.win_proc.append(process)
1128049
def record_process(self, process, prg=''):
        
        self._log(self.logFileProcess, force_to_string(process), prg)
536895
def RunOnce(self):
    
    start_time = time.time()
    processed = 0

    queue_manager = queue_manager_lib.QueueManager(token=self.token)
    for queue in self.queues:
      # Freezeing the timestamp used by queue manager to query/delete
      # notifications to avoid possible race conditions.
      queue_manager.FreezeTimestamp()

      fetch_messages_start = time.time()
      notifications = queue_manager.GetNotifications(queue)
      stats_collector_instance.Get().RecordEvent(
          "worker_time_to_retrieve_notifications",
          time.time() - fetch_messages_start)

      stuck_flows = []
      for n in notifications:
        if n.in_progress:
          stuck_flows.append(n)

      # Process stuck flows first
      if stuck_flows:
        self.ProcessStuckFlows(stuck_flows, queue_manager)

      notifications_available = []
      for notification in notifications:
        # Filter out session ids we already tried to lock but failed.
        if notification.session_id not in self.queued_flows:
          notifications_available.append(notification)

      try:
        # If we spent too much time processing what we have so far, the
        # active_sessions list might not be current. We therefore break here
        # so we can re-fetch a more up to date version of the list, and try
        # again later. The risk with running with an old active_sessions list
        # is that another worker could have already processed this message,
        # and when we try to process it, there is nothing to do - costing us a
        # lot of processing time. This is a tradeoff between checking the data
        # store for current information and processing out of date
        # information.
        processed += self.ProcessMessages(
            notifications_available, queue_manager,
            self.RUN_ONCE_MAX_SECONDS - (time.time() - start_time))

      # We need to keep going no matter what.
      except Exception as e:  # pylint: disable=broad-except
        logging.error("Error processing message %s. %s.", e,
                      traceback.format_exc())
        stats_collector_instance.Get().IncrementCounter("grr_worker_exceptions")
        if flags.FLAGS.pdb_post_mortem:
          pdb.post_mortem()

      queue_manager.UnfreezeTimestamp()
      # If we have spent too much time, stop.
      if (time.time() - start_time) > self.RUN_ONCE_MAX_SECONDS:
        return processed
    return processed
315157
def patch_python_logging_handlers():
    '''
    
    '''
    logging.StreamHandler = StreamHandler
    logging.FileHandler = FileHandler
    logging.handlers.SysLogHandler = SysLogHandler
    logging.handlers.WatchedFileHandler = WatchedFileHandler
    logging.handlers.RotatingFileHandler = RotatingFileHandler
    if sys.version_info >= (3, 2):
        logging.handlers.QueueHandler = QueueHandler
611591
def start(self, payload):
        """"""
        # Start specific processes, if `keys` is given in the payload
        if payload.get('keys'):
            succeeded = []
            failed = []
            for key in payload.get('keys'):
                success = self.process_handler.start_process(key)
                if success:
                    succeeded.append(str(key))
                else:
                    failed.append(str(key))

            message = ''
            if len(succeeded) > 0:
                message += 'Started processes: {}.'.format(', '.join(succeeded))
                status = 'success'
            if len(failed) > 0:
                message += '\nNo paused, queued or stashed process for keys: {}'.format(', '.join(failed))
                status = 'error'

            answer = {'message': message.strip(), 'status': status}

        # Start a all processes and the daemon
        else:
            self.process_handler.start_all()
            if self.paused:
                self.paused = False
                answer = {'message': 'Daemon and all processes started.',
                          'status': 'success'}
            else:
                answer = {'message': 'Daemon already running, starting all processes.',
                          'status': 'success'}
        return answer
494709
function ( payload ) {
		if ( payload.cmd === 'run' ) {

			var self = this.getParentScope();
			var callbacks = {
				callbackOnAssetAvailable: function ( payload ) {
					self.postMessage( payload );
				},
				callbackOnProgress: function ( text ) {
					if ( payload.logging.enabled && payload.logging.debug ) console.debug( 'WorkerRunner: progress: ' + text );
				}
			};

			// Parser is expected to be named as such
			var parser = new Parser();
			if ( typeof parser[ 'setLogging' ] === 'function' ) parser.setLogging( payload.logging.enabled, payload.logging.debug );
			this.applyProperties( parser, payload.params );
			this.applyProperties( parser, payload.materials );
			this.applyProperties( parser, callbacks );
			parser.workerScope = self;
			parser.parse( payload.data.input, payload.data.options );

			if ( payload.logging.enabled ) console.log( 'WorkerRunner: Run complete!' );

			callbacks.callbackOnAssetAvailable( {
				cmd: 'complete',
				msg: 'WorkerRunner completed run.'
			} );

		} else {

			console.error( 'WorkerRunner: Received unknown command: ' + payload.cmd );

		}
	}