The Pipeline tool.
197008
def main(**kwargs):
    """"""
    options = ApplicationOptions(**kwargs)
    Event.configure(is_logging_enabled=options.event_logging)
    application = Application(options)
    application.run(options.definition)
156275
def pipeline(names, steps):
    """"""
    steps, times = zip(*map(_maybe_timed, steps))
    fit_time = sum(times)
    if any(s is FIT_FAILURE for s in steps):
        fit_est = FIT_FAILURE
    else:
        fit_est = Pipeline(list(zip(names, steps)))
    return fit_est, fit_time
340860
def _pipeline_needs_fastq(config, data):
    
    aligner = config["algorithm"].get("aligner")
    support_bam = aligner in alignment.metadata.get("support_bam", [])
    return aligner and not support_bam
319310
function pipeS(...ms) {
  if(!arguments.length) {
    throw new TypeError(err)
  }

  const head =
    ms[0]

  if(!isSemigroupoid(head)) {
    throw new TypeError(err)
  }

  if(ms.length === 1) {
    return head
  }

  return ms.slice().reduce((comp, m) => {
    if(!isSameType(comp, m)) {
      throw new TypeError(err)
    }

    return comp.compose(m)
  })
}
723259
def matrix_worker(data):
    
    matrix = data['matrix']
    Logger.get_logger(__name__ + '.worker').info(
        "Processing pipeline for matrix entry '%s'", matrix['name'])

    env = matrix['env'].copy()
    env.update({'PIPELINE_MATRIX': matrix['name']})

    pipeline = Pipeline(model=data['model'], env=env, options=data['options'])
    pipeline.hooks = data['hooks']
    return pipeline.process(data['pipeline'])
361037
def run_pipelines(pipeline_id_pattern,
                  root_dir,
                  use_cache=True,
                  dirty=False,
                  force=False,
                  concurrency=1,
                  verbose_logs=True,
                  progress_cb=None,
                  slave=False):
    """"""
    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency,
                                               thread_name_prefix='T') as executor:
        try:
            results = []
            pending_futures = set()
            done_futures = set()
            finished_futures = []
            progress_thread = None
            progress_queue = None
            status_manager = status_mgr(root_dir)

            if progress_cb is not None:
                progress_queue = Queue()
                progress_thread = threading.Thread(target=progress_report_handler, args=(progress_cb, progress_queue))
                progress_thread.start()

            all_specs = specs_to_execute(pipeline_id_pattern, root_dir, status_manager, force, dirty, results)

            while True:

                done = None
                if len(done_futures) > 0:
                    done = done_futures.pop()
                    finished_futures.append(done)
                    done = done.result()[0]

                try:
                    spec = all_specs.send(done)
                except StopIteration:
                    spec = None

                if spec is None:
                    # Wait for all runners to idle...
                    if len(done_futures) == 0:
                        if len(pending_futures) > 0:
                            done_futures, pending_futures = \
                                concurrent.futures.wait(pending_futures,
                                                        return_when=concurrent.futures.FIRST_COMPLETED)
                            continue
                        else:
                            break
                    else:
                        continue

                if len(spec.validation_errors) > 0:
                    results.append(
                        ExecutionResult(spec.pipeline_id,
                                        False,
                                        {},
                                        ['init'] + list(map(str, spec.validation_errors)))
                    )
                    continue

                if slave:
                    ps = status_manager.get(spec.pipeline_id)
                    ps.init(spec.pipeline_details,
                            spec.source_details,
                            spec.validation_errors,
                            spec.cache_hash)
                    eid = gen_execution_id()
                    if ps.queue_execution(eid, 'manual'):
                        success, stats, errors = \
                            execute_pipeline(spec, eid,
                                             use_cache=use_cache)
                        results.append(ExecutionResult(
                            spec.pipeline_id,
                            success,
                            stats,
                            errors
                        ))
                    else:
                        results.append(
                            ExecutionResult(spec.pipeline_id,
                                            False,
                                            None,
                                            ['Already Running'])
                        )

                else:
                    f = executor.submit(remote_execute_pipeline,
                                        spec,
                                        root_dir,
                                        use_cache,
                                        verbose_logs,
                                        progress_queue)
                    pending_futures.add(f)

            for f in finished_futures:
                ret = f.result()
                results.append(ExecutionResult(*ret))

        except KeyboardInterrupt:
            pass
        finally:
            if slave:
                finalize()

            if progress_thread is not None:
                progress_queue.put(None)
                progress_thread.join()

    return results
547821
def _pipeline_output(self, pipeline, chunks, name):
        
        today = normalize_date(self.get_datetime())
        try:
            data = self._pipeline_cache.get(name, today)
        except KeyError:
            # Calculate the next block.
            data, valid_until = self.run_pipeline(
                pipeline, today, next(chunks),
            )
            self._pipeline_cache.set(name, data, valid_until)

        # Now that we have a cached result, try to return the data for today.
        try:
            return data.loc[today]
        except KeyError:
            # This happens if no assets passed the pipeline screen on a given
            # day.
            return pd.DataFrame(index=[], columns=data.columns)
1355132
def prepare_onetime_pipeline():
    """"""
    runner = ForemastRunner()
    runner.write_configs()
    runner.create_pipeline(onetime=os.getenv('ENV'))
    runner.cleanup()
695539
def _organize_tools_on(data, is_cwl):
    
    # want tools_on: [gvcf] if joint calling specified in CWL
    if is_cwl:
        if tz.get_in(["algorithm", "jointcaller"], data):
            val = tz.get_in(["algorithm", "tools_on"], data)
            if not val:
                val = []
            if not isinstance(val, (list, tuple)):
                val = [val]
            if "gvcf" not in val:
                val.append("gvcf")
            data["algorithm"]["tools_on"] = val
    return data
828473
def append(self, pipeline):
        
        for stage in pipeline.pipe:
            self._pipe.append(stage)
        return self
810562
def pipeline(self, transaction=True, shard_hint=None):
        '''  '''
        args_dict = {
            'connection_pool': self.connection_pool,
            'response_callbacks': self.response_callbacks,
            'transaction': transaction,
            'shard_hint': shard_hint,
            'logger': self.logger,
        }

        return StonePipeline(**args_dict)
773073
def to_pipeline
      pipeline = []
      pipeline.push({ "$skip" => skip }) if skip
      pipeline.push({ "$limit" => limit }) if limit
      pipeline.push({ "$sort" => sort }) if sort
      pipeline
    end
338258
def cleanup(self):
        """"""
        if self.data.hooks and len(self.data.hooks.cleanup) > 0:
            env = self.data.env_list[0].copy()
            env.update({'PIPELINE_RESULT': 'SUCCESS', 'PIPELINE_SHELL_EXIT_CODE': '0'})
            config = ShellConfig(script=self.data.hooks.cleanup, model=self.model,
                                 env=env, dry_run=self.options.dry_run,
                                 debug=self.options.debug, strict=self.options.strict,
                                 temporary_scripts_path=self.options.temporary_scripts_path)
            cleanup_shell = Bash(config)
            for line in cleanup_shell.process():
                yield line
724066
def add_after(self, pipeline):
        
        if not isinstance(pipeline, Pipeline):
            pipeline = Pipeline(pipeline)
        self.pipes = self.pipes[:] + pipeline.pipes[:]
        return self
1131907
def run(self, tokens):
        """"""
        for fn in self._stack:
            results = []
            for i, token in enumerate(tokens):
                # JS ignores additional arguments to the functions but we
                # force pipeline functions to declare (token, i, tokens)
                # or *args
                result = fn(token, i, tokens)
                if not result:
                    continue
                if isinstance(result, (list, tuple)):  # simulate Array.concat
                    results.extend(result)
                else:
                    results.append(result)
            tokens = results

        return tokens
624100
def prepare_app_pipeline():
    """"""
    runner = ForemastRunner()
    runner.write_configs()
    runner.create_app()
    runner.create_pipeline()
    runner.cleanup()
1118714
function compassFile(input, output) {
    task.src(input)
        .pipe(plumberPlugin())
        .pipe(connectPlugin.reload())
        .pipe(task.dest(output));
}