Put URL in queue, increase number of unfished tasks.
553674
def _put (self, url_data):
        """"""
        if self.shutdown or self.max_allowed_urls == 0:
            return
        log.debug(LOG_CACHE, "queueing %s", url_data.url)
        key = url_data.cache_url
        cache = url_data.aggregate.result_cache
        if url_data.has_result or cache.has_result(key):
            self.queue.appendleft(url_data)
        else:
            assert key is not None, "no result for None key: %s" % url_data
            if self.max_allowed_urls is not None:
                self.max_allowed_urls -= 1
            self.num_puts += 1
            if self.num_puts >= NUM_PUTS_CLEANUP:
                self.cleanup()
            self.queue.append(url_data)
        self.unfinished_tasks += 1
214229
def process_request(self, unused_request):
    """"""
    # Compare to the first half of toplevel() in context.py.
    tasklets._state.clear_all_pending()
    # Create and install a new context.
    ctx = tasklets.make_default_context()
    tasklets.set_context(ctx)
934493
def _append_task(t):
    
    task_queue = _get_task_queue()
    if t not in task_queue:
        log.debug('Appended new task to the queue: %s.', t)
        task_queue.append(t)
    else:
        log.debug('Did not append duplicate task to the queue: %s.', t)
    return None
833284
def create_queue(self, *args, **kwargs):
        """"""
        q = queue.Queue(*args, **kwargs)
        q.Empty = queue.Empty
        return q
431185
async def async_enqueue_download(self, resource):
        '''
        
        '''
        worker = self.pick_sticky(resource.url_string)
        await worker.enqueue(enums.Task.DOWNLOAD, (resource,))
138263
def put(self, target, callback = None, name = None, *args, **kargs):
        
        self.count_lock.acquire()
        if not self.shared['workers']:
            self.add(name = name)
        self.count_lock.release()
        self.queue.put_nowait((target, callback, name, args, kargs))
1244433
def _push_next(self):
        """"""
        r = next(self._iter, None)
        if r is None:
            return
        async_ret = self._worker_pool.apply_async(
            self._worker_fn, (r, self._batchify_fn, self._dataset))
        self._data_buffer[self._sent_idx] = async_ret
        self._sent_idx += 1
990795
function sendCommand(method, url, payload, skipQueue) {
    if (!self.connected) {
      debug('Queueing %s %s', method, url);
      return new P(function(resolve, reject) {
        var p = P.method(function() {
          debug('Dequeueing %s %s', method, url);
          return _sendCommand(method, url, payload)
            .then(resolve)
            .catch(reject);
        });
        if (skipQueue) {
          pendingCommands.splice(0, 0, p);
        } else {
          pendingCommands.push(p);
        }
      });
    }
    return _sendCommand(method, url, payload);
  }
356591
function queue(args) {
    args.push(resultsCallback);
    stack.push({caller: this.caller, args: args});
  }
108722
def add(self, task, multiprocess=False, processes=0):
        
        if self._first_task is None and hasattr(task, 'task_id'):
            self._first_task = task.task_id
        self.add_succeeded = True
        if multiprocess:
            queue = multiprocessing.Manager().Queue()
            pool = multiprocessing.Pool(processes=processes if processes > 0 else None)
        else:
            queue = DequeQueue()
            pool = SingleProcessPool()
        self._validate_task(task)
        pool.apply_async(check_complete, [task, queue])

        # we track queue size ourselves because len(queue) won't work for multiprocessing
        queue_size = 1
        try:
            seen = {task.task_id}
            while queue_size:
                current = queue.get()
                queue_size -= 1
                item, is_complete = current
                for next in self._add(item, is_complete):
                    if next.task_id not in seen:
                        self._validate_task(next)
                        seen.add(next.task_id)
                        pool.apply_async(check_complete, [next, queue])
                        queue_size += 1
        except (KeyboardInterrupt, TaskException):
            raise
        except Exception as ex:
            self.add_succeeded = False
            formatted_traceback = traceback.format_exc()
            self._log_unexpected_error(task)
            task.trigger_event(Event.BROKEN_TASK, task, ex)
            self._email_unexpected_error(task, formatted_traceback)
            raise
        finally:
            pool.close()
            pool.join()
        return self.add_succeeded
928614
def __spawn_new_request(self):
        

        first_in_line = self.queue.get_first(QueueItem.STATUS_QUEUED)
        
        if first_in_line is None:
            return False

        while self.routing.is_treshold_reached(first_in_line.request):
            self.queue.move(first_in_line, QueueItem.STATUS_CANCELLED)

            first_in_line = self.queue.get_first(QueueItem.STATUS_QUEUED)
            if first_in_line is None:
                return False

        self.__request_start(first_in_line)
        return True
379855
def run(delayed, concurrency, version_type=None, queue=None,
        raise_on_error=True):
    """"""
    if delayed:
        celery_kwargs = {
            'kwargs': {
                'version_type': version_type,
                'es_bulk_kwargs': {'raise_on_error': raise_on_error},
            }
        }
        click.secho(
            'Starting {0} tasks for indexing records...'.format(concurrency),
            fg='green')
        if queue is not None:
            celery_kwargs.update({'queue': queue})
        for c in range(0, concurrency):
            process_bulk_queue.apply_async(**celery_kwargs)
    else:
        click.secho('Indexing records...', fg='green')
        RecordIndexer(version_type=version_type).process_bulk_queue(
            es_bulk_kwargs={'raise_on_error': raise_on_error})
290653
def run_queue(self, options, todo):
    """"""
    utils.logging_debug('AutoBatcher(%s): %d items',
                        self._todo_tasklet.__name__, len(todo))
    batch_fut = self._todo_tasklet(todo, options)
    self._running.append(batch_fut)
    # Add a callback when we're done.
    batch_fut.add_callback(self._finished_callback, batch_fut, todo)
996270
private void addSinkTasks() {
    Runnable sinkTasks = new Runnable() {
      @Override
      public void run() {
        while (!metricsInSinkQueue.isEmpty()) {
          metricsSink.processRecord(metricsInSinkQueue.poll());
        }
      }
    };

    slaveLooper.addTasksOnWakeup(sinkTasks);
  }
230065
private void tryCreateExternalQueue(int index) {
        AuxState aux;
        if ((aux = auxState) != null && index >= 0) {
            WorkQueue q = new WorkQueue(this, null);
            q.config = index;
            q.scanState = ~UNSIGNALLED;
            q.qlock = 1;                   // lock queue
            boolean installed = false;
            aux.lock();
            try {                          // lock pool to install
                WorkQueue[] ws;
                if ((ws = workQueues) != null && index < ws.length &&
                    ws[index] == null) {
                    ws[index] = q;         // else throw away
                    installed = true;
                }
            } finally {
                aux.unlock();
            }
            if (installed) {
                try {
                    q.growArray();
                } finally {
                    q.qlock = 0;
                }
            }
        }
    }
860323
def schedule_next_requests(self):
        """"""
        # TODO: While there is capacity, schedule a batch of redis requests.
        for req in self.next_requests():
            self.crawler.engine.crawl(req, spider=self)
642044
function QueueEntry(url, success, error, queue) {
    var _this = this;

    // A list of callbacks that will be called. Supports multiple callbacks.
    this.callbacks = {
        success: [success],
        error: [error]
        /**
         * @type {string}
         */
    };this.url = url;
    /**
     * Passes all the arguments to LoadSuccess
     * @type {function}
     */
    this.success = function () {
        for (var _len = arguments.length, args = Array(_len), _key = 0; _key < _len; _key++) {
            args[_key] = arguments[_key];
        }

        runCallbacks(_this.callbacks.success, function (callback) {
            callback.apply(undefined, [url].concat(args));
        });
    };
    /**
     * @type {function}
     * @param {Error} errorObject
     */
    this.error = function (errorObject) {
        runCallbacks(_this.callbacks.success, function (callback) {
            callback(url, errorObject);
        });
    };

    /**
     * Cancels loading - removes the image
     */
    this.cancel = function () {
        queue.cancel(_this);
    };

    /**
     * @type {LoadTask|null}
     */
    this.task = null;
}