Bulk invoke a function via queues

    Uses internal implementation details of rq.
963227
def bulk_invoke(func, args, nargs):
    
    # for comparison, simplest thing that works
    # for i in nargs:
    #    argv = list(args)
    #    argv.append(i)
    #    func.delay(*argv)

    # some variances between cpy and pypy, sniff detect
    for closure in func.delay.func_closure:
        if getattr(closure.cell_contents, 'queue', None):
            ctx = closure.cell_contents
            break
    q = Queue(ctx.queue, connection=connection)
    argv = list(args)
    argv.append(None)
    job = Job.create(
        func, args=argv, connection=connection,
        description="bucket-%s" % func.func_name,
        origin=q.name, status=JobStatus.QUEUED, timeout=ctx.timeout,
        result_ttl=0, ttl=ctx.ttl)

    for n in chunks(nargs, 100):
        job.created_at = datetime.utcnow()
        with connection.pipeline() as pipe:
            for s in n:
                argv[-1] = s
                job._id = unicode(uuid4())
                job.args = argv
                q.enqueue_job(job, pipeline=pipe)
            pipe.execute()
1005537
public <T> void queue(StorageRequest<T> req, JsonBatchCallback<T> callback) throws IOException {
    checkState(
        !requestsExecutor.isShutdown() && !requestsExecutor.isTerminated(),
        "requestsExecutor should not be terminated to queue batch requests");
    if (maxRequestsPerBatch == 1) {
      responseFutures.add(
          requestsExecutor.submit(
              () -> {
                execute(req, callback);
                return null;
              }));
    } else {
      pendingRequests.add(batch -> req.queue(batch, callback));

      flushIfPossibleAndRequired();
    }
  }
128040
function(id, msg, result) {
        var q = queues[id], sc;
        if (q && q.onEnd) {
            sc = q.context || q;
            q.onEnd.call(sc, _returnData(q, msg, result));
        }
    }
725886
private void loadQueuedFlows() throws ExecutorManagerException {
    final List<Pair<ExecutionReference, ExecutableFlow>> retrievedExecutions =
        this.executorLoader.fetchQueuedFlows();
    if (retrievedExecutions != null) {
      for (final Pair<ExecutionReference, ExecutableFlow> pair : retrievedExecutions) {
        this.queuedFlows.enqueue(pair.getSecond(), pair.getFirst());
      }
    }
  }
69745
def multi_rpush(self, queue, values, bulk_size=0, transaction=False):
        '''  '''
        # Check that what we receive is iterable
        if hasattr(values, '__iter__'):
            self._multi_rpush_pipeline(self, queue, values, 0)
        else:
            raise ValueError('Expected an iterable')
353092
def bulk(self, request, *args, **kwargs):
        
        manager = self.get_manager()
        errors = {}
        preferences = []
        payload = request.data

        # first, we check updated preferences actually exists in the registry
        try:
            for identifier, value in payload.items():
                try:
                    preferences.append(
                        self.queryset.model.registry.get(identifier))
                except exceptions.NotFoundInRegistry:
                    errors[identifier] = 'invalid preference'
        except (TypeError, AttributeError):
            return Response('invalid payload', status=400)

        if errors:
            return Response(errors, status=400)

        # now, we generate an optimized Q objects to retrieve all matching
        # preferences at once from database
        queries = [
            Q(section=p.section.name, name=p.name)
            for p in preferences
        ]

        query = queries[0]
        for q in queries[1:]:
            query |= q
        preferences_qs = self.get_queryset().filter(query)

        # next, we generate a serializer for each database preference
        serializer_objects = []
        for p in preferences_qs:
            s = self.get_serializer_class()(
                p, data={'value': payload[p.preference.identifier()]})
            serializer_objects.append(s)

        validation_errors = {}

        # we check if any serializer is invalid
        for s in serializer_objects:
            if s.is_valid():
                continue
            validation_errors[s.instance.preference.identifier()] = s.errors

        if validation_errors:
            return Response(validation_errors, status=400)

        for s in serializer_objects:
            s.save()

        return Response(
            [s.data for s in serializer_objects],
            status=200,
        )
1110475
protected void executeQueuedCallbacks() {
        if (VERBOSE)
            Log.i(TAG, String.format("Executing %d queued callbacks", mCallbackQueue.size()));
        for (OAuthCallback cb : mCallbackQueue) {
            cb.onSuccess(getRequestFactoryFromCachedCredentials());
        }
    }
1207073
function processQueue(){
      try {
        // mark queue as processing to avoid concurrency
        processing = true;

        // process queue
        var item;
        while (item = queue.shift())
          item.fn.call(item.context);
      } finally {
        // queue is not processing anymore
        processing = false;

        // if any function in queue than exception was occured,
        // run rest functions in next code frame
        if (queue.length)
          timer = setImmediate(process);
      }
    }
1112800
@Override
  public Map<K, ValueHolder<V>> bulkCompute(final Set<? extends K> keys, final Function<Iterable<? extends Map.Entry<? extends K, ? extends V>>, Iterable<? extends Map.Entry<? extends K, ? extends V>>> remappingFunction)
      throws StoreAccessException {
    Map<K, ValueHolder<V>> valueHolderMap = new HashMap<>();
    if(remappingFunction instanceof Ehcache.PutAllFunction) {
      Ehcache.PutAllFunction<K, V> putAllFunction = (Ehcache.PutAllFunction<K, V>)remappingFunction;
      Map<K, V> entriesToRemap = putAllFunction.getEntriesToRemap();
      for(Map.Entry<K, V> entry: entriesToRemap.entrySet()) {
        PutStatus putStatus = silentPut(entry.getKey(), entry.getValue());
        if(putStatus == PutStatus.PUT) {
          putAllFunction.getActualPutCount().incrementAndGet();
          valueHolderMap.put(entry.getKey(), new ClusteredValueHolder<>(entry.getValue()));
        }
      }
    } else if(remappingFunction instanceof Ehcache.RemoveAllFunction) {
      Ehcache.RemoveAllFunction<K, V> removeAllFunction = (Ehcache.RemoveAllFunction<K, V>)remappingFunction;
      for (K key : keys) {
        boolean removed = silentRemove(key);
        if(removed) {
          removeAllFunction.getActualRemoveCount().incrementAndGet();
        }
      }
    } else {
      throw new UnsupportedOperationException("This bulkCompute method is not yet capable of handling generic computation functions");
    }
    return valueHolderMap;
  }
880738
function () {
    var result = Q.resolve();

    // loop through all promises created by the remote methods
    // this is synchronous, so it waits if a method is finished before
    // the next one will be executed
    this.actionQueue.forEach(function (f) {
      result = result.then(f);
    });

    // flush the queue & fire an event
    // when the queue finished its executions
    result.then(this.flushQueue.bind(this));
    return this;
  }
1005928
def invoke_in_mainloop(func, *args, **kwargs):
    
    results = queue.Queue()

    @gcall
    def run():
        try:
            data = func(*args, **kwargs)
            results.put(data)
            results.put(None)
        except BaseException:  # XXX: handle
            results.put(None)
            results.put(sys.exc_info())
            raise

    data = results.get()
    exception = results.get()

    if exception is None:
        return data
    else:
        tp, val, tb = results.get()
        raise tp, val, tb
301739
public BulkResponse
    executeBulkRequest(BulkRequestBuilder bulkRequestBuilder) {
        JMLog.info(log, "executeBulkRequest", bulkRequestBuilder);
        return bulkRequestBuilder.execute().actionGet();
    }
390023
def _multi_rpush_pipeline(self, pipe, queue, values, bulk_size=0):
        ''' 
        '''
        cont = 0
        for value in values:
            pipe.rpush(queue, value)
            if bulk_size != 0 and cont % bulk_size == 0:
                pipe.execute()
1170039
function (queueNumber, customParams, callback) {
      [queueNumber, customParams, callback] = doArgs(arguments, ['string|number', ['object', {}], 'function']);

      doRequest({
        urlPattern: [QUEUE_ITEM, queueNumber]
      }, customParams, callback);
    }
254604
function sqPush(queue, index, count, item) {
  queue[count] = item
  index[item]  = count
}
1049371
private void sendQueuedInvocations() {
        while (true) {
            MethodInvocation<?> currentRPC = queuedRpcList.poll();
            if (currentRPC == null) {
                return;
            }

            try {
                connector.executeAsyncMethod(currentRPC.getProxy(),
                                             currentRPC.getMethod(),
                                             currentRPC.getArgs(),
                                             currentRPC.getFuture());
            } catch (JoynrRuntimeException e) {
                currentRPC.getFuture().onFailure(e);
            } catch (Exception e) {
                currentRPC.getFuture().onFailure(new JoynrRuntimeException(e));
            }

        }

    }
327728
function queue(customParams, callback) {
      var _doArgs47 = doArgs(arguments, [['object', {}], 'function']);

      var _doArgs48 = _slicedToArray(_doArgs47, 2);

      customParams = _doArgs48[0];
      callback = _doArgs48[1];


      doRequest({
        urlPattern: [QUEUE]
      }, customParams, callback);
    }