Handle

Pass each queue item to the task handler, while remaining
within server memory and time limit constraints.
181157
protected function handle() {
		$this->lock_process();

		$shouldCancelAll = false;

		do {
			$batch = $this->get_batch();

			if ((count($batch->data) == 0) || (!$this->shouldHandle())) {
				$shouldCancelAll = true;
				Logger::info( "No batch to process", $batch);
				break;
			}

			Logger::info( "Processing Batch", $batch->data);
			foreach ( $batch->data as $key => $value ) {
				if ($this->shouldHandle()) {
					Logger::info( "Running task", $value);

					$task = $this->task( $value );

					if ( false !== $task ) {
						$batch->data[ $key ] = $task;
					} else {
						unset( $batch->data[ $key ] );
					}

					if ( $this->time_exceeded() || $this->memory_exceeded() ) {
						// Batch limits reached.
						break;
					}
				}
			}

			// Update or delete current batch.
			if ( ! empty( $batch->data ) ) {
				$this->update( $batch->key, $batch->data );
			} else {
				$this->delete( $batch->key );
			}
		} while ( ! $this->time_exceeded() && ! $this->memory_exceeded() && ! $this->is_queue_empty() );

		$this->unlock_process();

		if ($shouldCancelAll) {
			$this->complete();
			static::cancelAll();
		} else {
			if ( ! $this->is_queue_empty() ) {
				$this->dispatch();
			} else {
				$this->complete();
			}
		}

		wp_die();
	}
897129
def _handle_job_set(function):
    
    def call(self, job_set=taskhandle.NullJobSet()):
        job_set.started_job(str(self))
        function(self)
        job_set.finished_job()
    return call
470285
public void queueObject (@Nonnull final IIndexerWorkItem aItem)
  {
    ValueEnforcer.notNull (aItem, "Item");
    m_aImmediateCollector.queueObject (aItem);
  }
1353899
function processBuffer(flush, callback) {
    function taskCallback(err, payload) {
      if (err && !didError) {
        debug(`Queue task failed with error ${err.name}`);
        didError = true;
        q.kill();
        writer.emit('error', err);
      }
    }

    if (flush || buffer.length >= bufferSize) {
      // work through the buffer to break off bufferSize chunks
      // and feed the chunks to the queue
      do {
        // split the buffer into bufferSize chunks
        var toSend = buffer.splice(0, bufferSize);

        // and add the chunk to the queue
        debug(`Adding ${toSend.length} to the write queue.`);
        q.push({ docs: toSend }, taskCallback);
      } while (buffer.length >= bufferSize);

      // send any leftover documents to the queue
      if (flush && buffer.length > 0) {
        debug(`Adding remaining ${buffer.length} to the write queue.`);
        q.push({ docs: buffer }, taskCallback);
      }

      // wait until the queue size falls to a reasonable level
      async.until(
        // wait until the queue length drops to twice the paralellism
        // or until empty on the last write
        function() {
          // if we encountered an error, stop this until loop
          if (didError) {
            return true;
          }
          if (flush) {
            return q.idle() && q.length() === 0;
          } else {
            return q.length() <= parallelism * 2;
          }
        },

        function(cb) {
          setTimeout(cb, 20);
        },

        function() {
          if (flush && !didError) {
            writer.emit('finished', { total: written });
          }
          // callback when we're happy with the queue size
          callback();
        });
    } else {
      callback();
    }
  }
470276
void processTedQueue() {
		int totalProcessing = context.taskManager.calcWaitingTaskCountInAllChannels();
		if (totalProcessing >= TaskManager.LIMIT_TOTAL_WAIT_TASKS) {
			logger.warn("Total size of waiting tasks ({}) already exceeded limit ({}), skip this iteration (2)", totalProcessing, TaskManager.LIMIT_TOTAL_WAIT_TASKS);
			return;
		}
		Channel channel = context.registry.getChannelOrMain(Model.CHANNEL_QUEUE);
		int maxTask = context.taskManager.calcChannelBufferFree(channel);
		maxTask = Math.min(maxTask, 50);
		Map<String, Integer> channelSizes = new HashMap<String, Integer>();
		channelSizes.put(Model.CHANNEL_QUEUE, maxTask);
		List<TaskRec> heads = tedDao.reserveTaskPortion(channelSizes);
		if (heads.isEmpty())
			return;

		for (final TaskRec head : heads) {
			logger.debug("exec eventQueue for '{}', headTaskId={}", head.key1, head.taskId);
			channel.workers.execute(new TedRunnable(head) {
				@Override
				public void run() {
					processEventQueue(head);
				}
			});
		}
	}
551278
def set_server_callback(self, handle):
        
        if self.on_events:
            for event in self.on_events:
                handle.on_event(event, self.on_event)
        if self.on_changes:
            for change in self.on_changes:
                if change in ['patching', 'streaming']:
                    # Patch and stream events do not need handling on server
                    continue
                handle.on_change(change, self.on_change)
1154077
def handle(self):
        

        while True:
            data = self.request.recv(1024)
            if not data:
                break

            self.request.send(data)
        return True
213259
def serve(self):
        """"""
        while self.ready:
            try:
                self.tick()
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception:
                self.error_log(
                    'Error in HTTPServer.tick', level=logging.ERROR,
                    traceback=True,
                )

            if self.interrupt:
                while self.interrupt is True:
                    # Wait for self.stop() to complete. See _set_interrupt.
                    time.sleep(0.1)
                if self.interrupt:
                    raise self.interrupt
1326114
def handle(self):
    """"""
    # Mapper spec as form arguments.
    mapreduce_name = self._get_required_param("name")
    mapper_input_reader_spec = self._get_required_param("mapper_input_reader")
    mapper_handler_spec = self._get_required_param("mapper_handler")
    mapper_output_writer_spec = self.request.get("mapper_output_writer")
    mapper_params = self._get_params(
        "mapper_params_validator", "mapper_params.")
    params = self._get_params(
        "params_validator", "params.")

    # Default values.
    mr_params = map_job.JobConfig._get_default_mr_params()
    mr_params.update(params)
    if "queue_name" in mapper_params:
      mr_params["queue_name"] = mapper_params["queue_name"]

    # Set some mapper param defaults if not present.
    mapper_params["processing_rate"] = int(mapper_params.get(
        "processing_rate") or parameters.config.PROCESSING_RATE_PER_SEC)

    # Validate the Mapper spec, handler, and input reader.
    mapper_spec = model.MapperSpec(
        mapper_handler_spec,
        mapper_input_reader_spec,
        mapper_params,
        int(mapper_params.get("shard_count", parameters.config.SHARD_COUNT)),
        output_writer_spec=mapper_output_writer_spec)

    mapreduce_id = self._start_map(
        mapreduce_name,
        mapper_spec,
        mr_params,
        queue_name=mr_params["queue_name"],
        _app=mapper_params.get("_app"))
    self.json_response["mapreduce_id"] = mapreduce_id
653655
function(buffer, callback) {
    if (supportsWorkers) {
        // We have a worker queue, push an item into it and start processing
        workerQueue.push({ buffer: buffer, callback: callback });
        nextMd5Task();
    } else {
        // We don't have any Web Worker support,
        // queue an MD5 operation on the next tick
        process.nextTick(function() {
            callback(null, md5.ArrayBuffer.hash(buffer));
        });
    }
}
753115
def _consume(self):
        
        while True:
            try:
                task, args, kwargs = self.queue.get(True)
                task(*args, **kwargs)
            finally:
                self.queue.task_done()
497220
def _process_queue(self):
        '''
        
        '''
        if len(self._queue):
            args, kwargs = self._queue.popleft()
            self.publish(*args, **kwargs)
1016528
void enqueue(ServerRequest request) {
        synchronized (reqQueueLockObject) {
            if (request != null) {
                queue.add(request);
                if (getSize() >= MAX_ITEMS) {
                    queue.remove(1);
                }
                persist();
            }
        }
    }
422854
public void run() {
        Runnable r = null;
        while (true) {
            // Try to drain the maximum capacity of thread-local items, checking
            // whether any were available
            if (workQueue.drainTo(internalQueue, threadLocalItems) == 0) {
                // block until a work item is available
                try {
                    internalQueue.offer(workQueue.take());
                } catch (InterruptedException ie) {
                    throw new Error(ie);
                }
            }
            // Execute all of the thread-local items
            while ((r = internalQueue.poll()) != null)
                r.run();
        }
    }
1263622
def process_item(self, item):
        
        logger.debug("Processing item: %s (queue size: %s)", item,
                     self.queue.qsize())
        method, kwargs = item

        if method not in self.supported_methods:
            raise Error("Not expected service method: {}".format(method))

        try:
            if method == "log":
                self.process_log(**kwargs)
            else:
                self._post_log_batch()
                getattr(self.rp_client, method)(**kwargs)
        except Exception:
            if self.error_handler:
                self.error_handler(sys.exc_info())
            else:
                self.terminate(nowait=True)
                raise
553661
def submit_task(rel_path, cache_string, buffer):
    """"""
    global upload_queue
    global upload_thread
    upload_queue.put((rel_path, cache_string, buffer))

    if upload_thread is None or not upload_thread.is_alive():
        upload_thread = UploaderThread()
        upload_thread.start()
507053
def handle(self):
        
        logger = StreamHandler.logger
        logger.debug("handling requests with message handler %s " % StreamHandler.message_handler.__class__.__name__)

        message_handler = StreamHandler.message_handler

        try:
            while True:
                logger.debug('waiting for more data')
                if not message_handler.handle(self.request, StreamHandler.BUFFER_SIZE):
                    break

            logger.warning("connection closed from %s" % (self.client_address[0]))
            self.request.close()
        except:
            logger.exception("connection closed from %s" % (self.client_address[0]))
        finally:
            self.request.close()