Get a task from queue when bucket available
491016
def get(self):
        ''''''
        if self.bucket.get() < 1:
            return None
        now = time.time()
        self.mutex.acquire()
        try:
            task = self.priority_queue.get_nowait()
            self.bucket.desc()
        except Queue.Empty:
            self.mutex.release()
            return None
        task.exetime = now + self.processing_timeout
        self.processing.put(task)
        self.mutex.release()
        return task.taskid
1009280
def _get_async(self, **q_options):
    """"""
    res = yield self.fetch_async(1, **q_options)
    if not res:
      raise tasklets.Return(None)
    raise tasklets.Return(res[0])
1340617
def get_tasks():
    ''''''
    return {
        name: get_task_queue(name, cls)
        for name, cls in celery.tasks.items()
        # Exclude celery internal tasks
        if not name.startswith('celery.')
        # Exclude udata test tasks
        and not name.startswith('test-')
    }
845066
def to_task(self):
        """"""
        from google.appengine.api.taskqueue import Task

        task_args = self.get_task_args().copy()

        payload = None
        if 'payload' in task_args:
            payload = task_args.pop('payload')

        kwargs = {
            'method': METHOD_TYPE,
            'payload': json.dumps(payload)
        }

        kwargs.update(task_args)

        return Task(**kwargs)
801761
def get(self):
        ''''''
        now = time.time()
        if self.bucket >= self.burst:
            self.last_update = now
            return self.bucket
        bucket = self.rate * (now - self.last_update)
        self.mutex.acquire()
        if bucket > 1:
            self.bucket += bucket
            if self.bucket > self.burst:
                self.bucket = self.burst
            self.last_update = now
        self.mutex.release()
        return self.bucket
992076
def send_task(self, task, force=True):
        '''
        
        '''
        try:
            self.out_queue.put_nowait(task)
        except Queue.Full:
            if force:
                self._send_buffer.appendleft(task)
            else:
                raise
498659
def _pool_get(get, results, next_available, task_next_lock, to_skip, \
                  task_num, pool_size, id_self):
        
        log.debug('NuMap(%s) started pool_getter' % id_self)
        # should return when all workers have returned, each worker sends a 
        # sentinel before returning. Before returning it should send sentinels to
        # all tasks but the next available queue should be released only if we 
        # know that no new results will arrive.
        sentinels = 0
        result_ids, last_result_id, very_last_result_id = {}, {}, {}
        for i in xrange(task_num):
            last_result_id[i] = -1
            very_last_result_id[i] = -2
            result_ids[i] = set()

        while True:
            try:
                log.debug('NuMap(%s) pool_getter waits for a result.' % id_self)
                #gc.disable()
                result = get()
                #gc.enable()
            except (IOError, EOFError):
                log.error('NuMap(%s) pool_getter has a pipe problem.' % id_self)
                break

            # got a sentinel?
            if result is None:
                sentinels += 1
                log.debug('NuMap(%s) pool_getter got a sentinel.' % id_self)
                if sentinels == pool_size:
                    log.debug('NuMap(%s) pool_getter got all sentinels.' % \
                              id_self)
                    # here we are escaping.
                    break
                else:
                    # waiting for more sentinels or results to come.
                    continue

            # got a sentinel for a task?
            # only one sentinel per task will be received
            if result[1] is None:
                task = result[0]
                very_last_result_id[task] = result[2]
                if last_result_id[task] == very_last_result_id[task]:
                    results[task].put(('stop', False, 'stop'))
                    next_available[task].put(True)
                    log.debug('NuMap(%s) pool_getter sent sentinel for task %s.'\
                                                            % (id_self, task))
                continue

            # got some result for some task, which might be an exception
            task, i, is_valid, real_result = result
            # locked if next for this task is in 
            # the process of raising a TimeoutError
            task_next_lock[task].acquire()
            log.debug('NuMap(%s) pool_getter received result %s for task %s)' % \
                      (id_self, i, task))

            if to_skip[task]:
                log.debug('NuMap(%s) pool_getter skips results: %s' % (id_self, \
                range(last_result_id[task] + 1, last_result_id[task] + \
                      to_skip[task] + 1)))
                last_result_id[task] += to_skip[task]
                to_skip[task] = 0

            if i > last_result_id[task]:
                result_ids[task].add(i)
                results[task].put((i, is_valid, real_result))
                log.debug('NuMap(%s) pool_getter put result %s for task %s to queue' % \
                          (id_self, i, task))
            else:
                log.debug('NuMap(%s) pool_getter skips result %s for task %s' % \
                          (id_self, i, task))

            # this releases the next method for each ordered result in the queue
            # if the NuMap instance is ordered =False this information is 
            # ommitted.
            while last_result_id[task] + 1 in result_ids[task]:
                next_available[task].put(True)
                last_result_id[task] += 1
                log.debug('NuMap(%s) pool_getter released task: %s' % \
                          (id_self, task))
            if last_result_id[task] == very_last_result_id[task]:
                results[task].put(('stop', False, 'stop'))
                next_available[task].put(True)
            # release the next method
            task_next_lock[task].release()
        log.debug('NuMap(%s) pool_getter returns' % id_self)
191024
def _process_queue_tasks(self, queue, queue_lock, task_ids, now, log):
        """"""

        processed_count = 0

        # Get all tasks
        serialized_tasks = self.connection.mget([
            self._key('task', task_id) for task_id in task_ids
        ])

        # Parse tasks
        tasks = []
        for task_id, serialized_task in zip(task_ids, serialized_tasks):
            if serialized_task:
                task_data = json.loads(serialized_task)
            else:
                # In the rare case where we don't find the task which is
                # queued (see ReliabilityTestCase.test_task_disappears),
                # we log an error and remove the task below. We need to
                # at least initialize the Task object with an ID so we can
                # remove it.
                task_data = {'id': task_id}

            task = Task(self.tiger, queue=queue, _data=task_data,
                        _state=ACTIVE, _ts=now)

            if not serialized_task:
                # Remove task as per comment above
                log.error('not found', task_id=task_id)
                task._move()
            elif task.id != task_id:
                log.error('task ID mismatch', task_id=task_id)
                # Remove task
                task._move()
            else:
                tasks.append(task)

        # List of task IDs that exist and we will update the heartbeat on.
        valid_task_ids = set(task.id for task in tasks)

        # Group by task func
        tasks_by_func = OrderedDict()
        for task in tasks:
            func = task.serialized_func
            if func in tasks_by_func:
                tasks_by_func[func].append(task)
            else:
                tasks_by_func[func] = [task]

        # Execute tasks for each task func
        for tasks in tasks_by_func.values():
            success, processed_tasks = self._execute_task_group(queue,
                    tasks, valid_task_ids, queue_lock)
            processed_count = processed_count + len(processed_tasks)
            log.debug('processed', attempted=len(tasks),
                      processed=processed_count)
            for task in processed_tasks:
                self._finish_task_processing(queue, task, success)

        return processed_count
157409
def wait_for_task(task_data, task_uri='/tasks'):
    
    taskid = post_task(task_data, task_uri)

    if isinstance(task_data, str):
        json_data = json.loads(task_data)
    else:
        json_data = task_data

    # inspect the task to see if a timeout is configured
    job = json_data['job'][0]
    env = job.get('credentials')
    task_type = job.get('type')

    timeout = TASK_TIMEOUTS.get(env, dict()).get(task_type, DEFAULT_TASK_TIMEOUT)

    LOG.debug("Task %s will timeout after %s", task_type, timeout)

    return check_task(taskid, timeout)
913423
function iterateOverCallbacks(bucket, args) {
    bucket.forEach(function iterator(callback) {
        nextTick(function queuedCallback() {
            callback.apply(null, args);
        });
    });
}
1285794
def enq(self, task):
        
        if not self._stop.is_set():
            self._inq.put(task)
        else:
            raise RuntimeError("Attempted to enqueue an operation while "
                               "multi pool was shutdown!")
53811
function queue(task, done) {
    var storage = this,
        key = this.keygen(task);

    // store task as JSON serialized string
    task = JSON.stringify(task);

    // first add the key to the queue
    this.redis().rpush(this.queueKey(), key, function(err) {
        if (err) done(err);

        // now store task data
        else storage.redis().set(key, task, function(err) {
            if (err) done(err);
            else done(null, key);
        });
    });
}
673348
def get(self, block=True, timeout=None):
        """"""
        return self._queue.get(block, timeout)
1035879
def handle_finish(queue_name):
    """"""
    task_id = request.form.get('task_id', type=str)
    owner = request.form.get('owner', request.remote_addr, type=str)
    error = request.form.get('error', type=str) is not None
    try:
        work_queue.finish(queue_name, task_id, owner, error=error)
    except work_queue.Error, e:
        return utils.jsonify_error(e)

    db.session.commit()
    logging.debug('Task finished: queue=%r, task_id=%r, owner=%r, error=%r',
                  queue_name, task_id, owner, error)
    return flask.jsonify(success=True)
1020009
private void queueOrOfferMPTask(TransactionTask task)
    {
        // The pending map will only have an entry for the transaction if the first fragment is
        // still pending durability.
        Queue<TransactionTask> pendingTasks = m_mpsPendingDurability.get(task.getTxnId());
        if (pendingTasks != null) {
            pendingTasks.offer(task);
        } else {
            m_pendingTasks.offer(task);
        }
    }
1153656
def ProcessTask(self, target, args, name, queueing_time):
    """"""

    if self.pool.name:
      time_in_queue = time.time() - queueing_time
      stats_collector_instance.Get().RecordEvent(
          _QUEUEING_TIME_METRIC, time_in_queue, fields=[self.pool.name])

      start_time = time.time()
    try:
      target(*args)
    # We can't let a worker die because one of the tasks it has to process
    # throws an exception. Therefore, we catch every error that is
    # raised in the call to target().
    except Exception as e:  # pylint: disable=broad-except
      if self.pool.name:
        stats_collector_instance.Get().IncrementCounter(
            _TASK_EXCEPTIONS_METRIC, fields=[self.pool.name])
      logging.exception("Caught exception in worker thread (%s): %s", name,
                        str(e))

    if self.pool.name:
      total_time = time.time() - start_time
      stats_collector_instance.Get().RecordEvent(
          _WORKING_TIME_METRIC, total_time, fields=[self.pool.name])
748043
function queue(runner) {
    if (running < tasks) {
      ++running;
      runner(release);
    } else {
      pending.push(runner);
    }
  }