Check if a work input needs processing for parallelization.
972587
def bam_needs_processing(data):
    
    return ((data.get("work_bam") or data.get("align_bam")) and
            (any(tz.get_in(["config", "algorithm", x], data) for x in
                 ["variantcaller", "mark_duplicates", "recalibrate", "realign", "svcaller",
                  "jointcaller", "variant_regions"])
             or any(k in data for k in ["cwl_keys", "output_cwl_keys"])))
586585
def async_comp_check(self, original, loc, tokens):
        """"""
        return self.check_py("36", "async comprehension", original, loc, tokens)
1291617
def process(self, document):
        """"""
        self.logger.info("Processing group of tasks (parallel=%s)", self.get_parallel_mode())
        self.pipeline.data.env_list[2] = {}

        output, shells = [], []
        result = Adapter({'success': True, 'output': []})
        for task_entry in document:
            key, entry = list(task_entry.items())[0]

            if (not self.parallel or key == 'env') and len(shells) > 0:
                result = Adapter(self.process_shells(shells))
                output += result.output
                shells = []
                if not result.success:
                    break

            if key == 'env':
                self.pipeline.data.env_list[2].update(entry)

            elif key in ['shell', 'docker(container)', 'docker(image)', 'python',
                         'packer', 'ansible(simple)']:
                self.prepare_shell_data(shells, key, entry)

        if result.success:
            result = Adapter(self.process_shells(shells))
            output += result.output

        self.event.delegate(result.success)
        return {'success': result.success, 'output': output}
335188
def evaluate(self, sequence, transformations):
        
        result = sequence
        parallel = partial(
            parallelize, processes=self.processes, partition_size=self.partition_size)
        staged = []
        for transform in transformations:
            strategies = transform.execution_strategies or {}
            if ExecutionStrategies.PARALLEL in strategies:
                staged.insert(0, transform.function)
            else:
                if staged:
                    result = parallel(compose(*staged), result)
                    staged = []
                if ExecutionStrategies.PRE_COMPUTE in strategies:
                    result = list(result)
                result = transform.function(result)
        if staged:
            result = parallel(compose(*staged), result)
        return iter(result)
661057
def check_jobs(jobs):
    """"""
    if jobs == 0:
        raise click.UsageError("Jobs must be >= 1 or == -1")
    elif jobs < 0:
        import multiprocessing

        jobs = multiprocessing.cpu_count()
    return jobs
1236314
def _check(sample, data):
    """"""
    if dd.get_chip_method(sample).lower() == "atac":
        return [sample]
    if dd.get_phenotype(sample) == "input":
        return None
    for origin in data:
        if dd.get_batch(sample) in (dd.get_batches(origin[0]) or []) and dd.get_phenotype(origin[0]) == "input":
            sample["work_bam_input"] = origin[0].get("work_bam")
            return [sample]
    return [sample]
765532
boolean shouldDrainBuffers(boolean delayable) {
        if (executor.isShutdown()) {
            DrainStatus status = drainStatus.get();
            return (status != PROCESSING) && (!delayable || (status == REQUIRED));
        }
        return false;
    }
507318
def check_async(paths, options, rootdir=None):
    
    LOGGER.info('Async code checking is enabled.')
    path_queue = Queue.Queue()
    result_queue = Queue.Queue()

    for num in range(CPU_COUNT):
        worker = Worker(path_queue, result_queue)
        worker.setDaemon(True)
        LOGGER.info('Start worker #%s', (num + 1))
        worker.start()

    for path in paths:
        path_queue.put((path, dict(options=options, rootdir=rootdir)))

    path_queue.join()

    errors = []
    while True:
        try:
            errors += result_queue.get(False)
        except Queue.Empty:
            break

    return errors
1229037
function () {
                    //Check if any input was added dynamically
                    if (job.input.length > 0) {
                        job.is_complete = false;
                        return false;
                    }

                    //Wait for workers or instances that are still working
                    return worker_count > 0 ? areWorkersComplete() : job.instances <= 0;
                }
911547
function (input) {
            if (typeof input !== 'undefined' && input !== null && input !== false) {
                master.emit('input', input, for_worker);
            } else {

                //No input? We might be done..

                var isComplete = function () {
                    //Check if any input was added dynamically
                    if (job.input.length > 0) {
                        job.is_complete = false;
                        return false;
                    }

                    //Wait for workers or instances that are still working
                    return worker_count > 0 ? areWorkersComplete() : job.instances <= 0;
                };

                //If we're not complete, check periodically
                if (isComplete()) {
                    master.emit('complete');
                } else {
                    completeCheckInterval = setInterval(function () {
                        if (isComplete()) {
                            clearInterval(completeCheckInterval);
                            master.emit('complete');
                        }
                    }, 300);
                }
            }
        }
350370
def usable_cpu_count():
    
    try:
        result = len(os.sched_getaffinity(0))
    except AttributeError:
        try:
            result = len(psutil.Process().cpu_affinity())
        except AttributeError:
            result = os.cpu_count()
    return result
898464
def assert_perform_under(threshold, options = {}, &work)
      actual, _ = ExecutionTime.run(options, &work)
      actual <= threshold
    end
565343
def cpu_count(logical=True):
    
    if logical:
        from multiprocessing import cpu_count
        ncpu=cpu_count()
    else:
        import psutil
        ncpu=psutil.cpu_count(logical=False)
    return ncpu
1328933
@Override
    public boolean supportsParallelProcessing() {
        try {
            return getSettings().getBoolean(Settings.KEYS.ANALYZER_ARTIFACTORY_PARALLEL_ANALYSIS, true);
        } catch (InvalidSettingException ex) {
            LOGGER.debug("Invalid setting for analyzer.artifactory.parallel.analysis; using true.");
        }
        return true;
    }
797620
def process_request(self, request, client_address):
        """"""
        if not self._parallel:
            self._process_request(request, client_address)
            return
        t = self._thread_factory(
            target=self._process_request, args=(request, client_address))
        t.daemon = True
        t.start()
1022920
private boolean tryCompensate(WorkQueue w) {
        boolean canBlock; int wl;
        long c = ctl;
        WorkQueue[] ws = workQueues;
        int pc = config & SMASK;
        int ac = pc + (int)(c >> AC_SHIFT);
        int tc = pc + (short)(c >> TC_SHIFT);
        if (w == null || w.qlock < 0 || pc == 0 ||  // terminating or disabled
            ws == null || (wl = ws.length) <= 0)
            canBlock = false;
        else {
            int m = wl - 1, sp;
            boolean busy = true;                    // validate ac
            for (int i = 0; i <= m; ++i) {
                int k; WorkQueue v;
                if ((k = (i << 1) | 1) <= m && k >= 0 && (v = ws[k]) != null &&
                    v.scanState >= 0 && v.currentSteal == null) {
                    busy = false;
                    break;
                }
            }
            if (!busy || ctl != c)
                canBlock = false;                   // unstable or stale
            else if ((sp = (int)c) != 0)            // release idle worker
                canBlock = tryRelease(c, ws[m & sp], 0L);
            else if (tc >= pc && ac > 1 && w.isEmpty()) {
                long nc = ((AC_MASK & (c - AC_UNIT)) |
                           (~AC_MASK & c));         // uncompensated
                canBlock = U.compareAndSwapLong(this, CTL, c, nc);
            }
            else if (tc >= MAX_CAP ||
                     (this == common && tc >= pc + COMMON_MAX_SPARES))
                throw new RejectedExecutionException(
                    "Thread limit exceeded replacing blocked worker");
            else {                                  // similar to tryAddWorker
                boolean isSpare = (tc >= pc);
                long nc = (AC_MASK & c) | (TC_MASK & (c + TC_UNIT));
                canBlock = (U.compareAndSwapLong(this, CTL, c, nc) &&
                            createWorker(isSpare)); // throws on exception
            }
        }
        return canBlock;
    }
242535
def run(self):
        
        try:
            while True:
                input_chunks = [input.get() for input in self.input_queues]
                for input in self.input_queues:
                    input.task_done()
                if any(chunk is QUEUE_ABORT for chunk in input_chunks):
                    self.abort()
                    return
                if any(chunk is QUEUE_FINISHED for chunk in input_chunks):
                    break
                self.output(self.process_chunks(input_chunks))
            # Finalise the final chunk (process_chunks does this for all
            # but the last chunk).
            self.output(self.finalise())
        except:
            self.abort()
            raise
        else:
            for queue in self.output_queues:
                queue.put(QUEUE_FINISHED)