Retrieve CWL inputs shared amongst different workflows.
970422
def _cwl_workflow_template(inputs, top_level=False):
    
    ready_inputs = []
    for inp in inputs:
        cur_inp = copy.deepcopy(inp)
        for attr in ["source", "valueFrom", "wf_duplicate"]:
            cur_inp.pop(attr, None)
        if top_level:
            cur_inp = workflow._flatten_nested_input(cur_inp)
        cur_inp = _clean_record(cur_inp)
        ready_inputs.append(cur_inp)
    return {"class": "Workflow",
            "cwlVersion": "v1.0",
            "hints": [],
            "requirements": [{"class": "EnvVarRequirement",
                              "envDef": [{"envName": "MPLCONFIGDIR", "envValue": "."}]},
                             {"class": "ScatterFeatureRequirement"},
                             {"class": "SubworkflowFeatureRequirement"}],
            "inputs": ready_inputs,
            "outputs": [],
            "steps": []}
545458
def get(self):
        
        return WorkflowStatisticsContext(
            self._version,
            workspace_sid=self._solution['workspace_sid'],
            workflow_sid=self._solution['workflow_sid'],
        )
197263
def _convert_to_cwl_json(data, fnargs, input_files):
    
    out = {}
    for outvar in _get_output_cwl_keys(fnargs):
        keys = []
        for key in outvar.split("__"):
            try:
                key = int(key)
            except ValueError:
                pass
            keys.append(key)
        if isinstance(data, dict):
            out[outvar] = _to_cwl(tz.get_in(keys, data), input_files)
        else:
            out[outvar] = [_to_cwl(tz.get_in(keys, x), input_files) for x in data]
    return out
161707
def _extract_from_subworkflow(vs, step):
    
    substep_ids = set([x.name for x in step.workflow])
    out = []
    for var in vs:
        internal = False
        parts = var["id"].split("/")
        if len(parts) > 1:
            if parts[0] in substep_ids:
                internal = True
        if not internal:
            var.pop("source", None)
            out.append(var)
    return out
1361606
def getWSDL(self):
        """"""
        parent = self
        while 1:
            # skip any collections
            if isinstance(parent, WSDL):
                return parent
            try: parent = parent.parent()
            except: break
            
        return None
585114
def execute(client, output_file, output_paths=None):
    """"""
    output_paths = output_paths or set()

    import cwltool.factory
    from cwltool import workflow
    from cwltool.context import LoadingContext, RuntimeContext
    from cwltool.utils import visit_class

    def construct_tool_object(toolpath_object, *args, **kwargs):
        """Fix missing locations."""
        protocol = 'file://'

        def addLocation(d):
            if 'location' not in d and 'path' in d:
                d['location'] = protocol + d['path']

        visit_class(toolpath_object, ('File', 'Directory'), addLocation)
        return workflow.default_make_tool(toolpath_object, *args, **kwargs)

    argv = sys.argv
    sys.argv = ['cwltool']

    # Keep all environment variables.
    runtime_context = RuntimeContext(
        kwargs={
            'rm_tmpdir': False,
            'move_outputs': 'leave',
            'preserve_entire_environment': True,
        }
    )
    loading_context = LoadingContext(
        kwargs={
            'construct_tool_object': construct_tool_object,
        }
    )

    factory = cwltool.factory.Factory(
        loading_context=loading_context,
        runtime_context=runtime_context,
    )
    process = factory.make(os.path.relpath(str(output_file)))
    outputs = process()

    sys.argv = argv

    # Move outputs to correct location in the repository.
    output_dirs = process.factory.executor.output_dirs

    def remove_prefix(location, prefix='file://'):
        if location.startswith(prefix):
            return location[len(prefix):]
        return location

    locations = {
        remove_prefix(output['location'])
        for output in outputs.values()
    }

    with progressbar(
        locations,
        label='Moving outputs',
    ) as bar:
        for location in bar:
            for output_dir in output_dirs:
                if location.startswith(output_dir):
                    output_path = location[len(output_dir):].lstrip(
                        os.path.sep
                    )
                    destination = client.path / output_path
                    if destination.is_dir():
                        shutil.rmtree(str(destination))
                        destination = destination.parent
                    shutil.move(location, str(destination))
                    continue

    unchanged_paths = client.remove_unmodified(output_paths)
    if unchanged_paths:
        click.echo(
            'Unchanged files:\n\n\t{0}'.format(
                '\n\t'.join(
                    click.style(path, fg='yellow') for path in unchanged_paths
                )
            )
        )
333390
def _load_cwr_defaults(self):
        
        if self._cwr_defaults is None:
            self._cwr_defaults = self._reader.read_yaml_file(
                self._file_defaults)

        return self._cwr_defaults
953007
def _merge_cwlinputs(items_by_key, input_order, parallel):
    
    items_by_key = _maybe_nest_bare_single(items_by_key, parallel)
    if parallel == "multi-combined":
        items_by_key, input_order = _concat_records(items_by_key, input_order)
    var_items = set([_item_count(items_by_key[tuple(k.split("__"))])
                     for (k, t) in input_order.items() if t == "var"])
    rec_items = set([_item_count(items_by_key[k]) for (k, t) in input_order.items() if t == "record"])
    if var_items:
        num_items = var_items
        if len(num_items) == 2 and 1 in num_items:
            num_items.remove(1)
            items_by_key_test = _check_for_single_nested(num_items.pop(), items_by_key, input_order)
            var_items = set([_item_count(items_by_key_test[tuple(k.split("__"))])
                             for (k, t) in input_order.items() if t == "var"])
            num_items = var_items
        assert len(num_items) == 1, "Non-consistent variable data counts in CWL input:\n%s" % \
            (pprint.pformat(items_by_key))
        items_by_key, num_items = _nest_vars_in_rec(var_items, rec_items, input_order, items_by_key, parallel)
    else:
        num_items = rec_items
        assert len(num_items) == 1, "Non-consistent record data counts in CWL input:\n%s" % \
            (pprint.pformat(items_by_key))
    target_items = num_items.pop()
    out = [{} for _ in range(target_items)]
    for (cwl_key, cwl_type) in input_order.items():
        if cwl_type == "var":
            cwl_key = tuple(cwl_key.split("__"))
        cur_vals = items_by_key[cwl_key]
        if _is_nested_single(cur_vals, target_items):
            cur_vals = [[x] for x in cur_vals[0]]
        for i, cur_val in enumerate(cur_vals):
            if isinstance(cwl_key, (list, tuple)):
                # nested batches with records
                if (parallel.startswith(("batch", "multi-parallel")) and
                      isinstance(out[i], (list, tuple))):
                    for j in range(len(out[i])):
                        out[i][j] = _update_nested(list(cwl_key), cur_val, out[i][j], allow_overwriting=True)
                else:
                    out[i] = _update_nested(list(cwl_key), cur_val, out[i], allow_overwriting=True)
            elif out[i] == {}:
                out[i] = cur_val
            else:
                # Handle single non-batched records
                if isinstance(cur_val, (list, tuple)) and len(cur_val) == 1:
                    cur_val = cur_val[0]
                assert isinstance(cur_val, dict), (cwl_key, cur_val)
                for k, v in cur_val.items():
                    out[i] = _update_nested([k], v, out[i], allow_overwriting=True)
    return out
981869
def GetInput(self):
    """"""
    clients = GetAllClients(token=self.token)
    logging.debug("Got %d clients", len(clients))
    return clients
1249008
def get_workflow_graph(promise):
    """"""
    workflow = get_workflow(promise)

    dot = Digraph()
    for i, n in workflow.nodes.items():
        dot.node(str(i), label="{0} \n {1}".format(
            n.foo.__name__,
            _format_arg_list(n.bound_args.args)))

    for i in workflow.links:
        for j in workflow.links[i]:
            dot.edge(str(i), str(j[0]), label=str(j[1].name))

    return dot
176454
def getWorkerInfo(dataTask):
    """"""
    workertime = []
    workertasks = []
    for fichier, vals in dataTask.items():
        if hasattr(vals, 'values'):
            #workers_names.append(fichier)
            # Data from worker
            totaltime = sum([a['executionTime'] for a in vals.values()])
            totaltasks = sum([1 for a in vals.values()])
            workertime.append(totaltime)
            workertasks.append(totaltasks)
    return workertime, workertasks
48993
def system_find_global_workflows(input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/system/findGlobalWorkflows', input_params, always_retry=always_retry, **kwargs)
374091
def from_cwl(cls, data, __reference__=None):
        """"""
        class_name = data.get('class', None)
        cls = cls.registry.get(class_name, cls)

        if __reference__:
            with with_reference(__reference__):
                self = cls(
                    **{k: v
                       for k, v in iteritems(data) if k != 'class'}
                )
        else:
            self = cls(**{k: v for k, v in iteritems(data) if k != 'class'})
        return self
306840
def show_inputs(client, workflow):
    """"""
    for input_ in workflow.inputs:
        click.echo(
            '{id}: {default}'.format(
                id=input_.id,
                default=_format_default(client, input_.default),
            )
        )
    sys.exit(0)
7690
def retrieveVals(self):
        """"""
        proc_info = ProcessInfo()
        stats = {}
        for (prefix, is_thread) in (('proc', False), 
                                    ('thread', True)):
            graph_name = '%s_status' % prefix
            if self.hasGraph(graph_name):
                if not stats.has_key(prefix):
                    stats[prefix] = proc_info.getProcStatStatus(is_thread)
                for (fname, stat_key) in (
                    ('unint_sleep', 'uninterruptable_sleep'),
                    ('stopped', 'stopped'),
                    ('defunct', 'defunct'),
                    ('running', 'running'),
                    ('sleep', 'sleep')):
                    self.setGraphVal(graph_name, fname, 
                                     stats[prefix]['status'].get(stat_key))
            graph_name = '%s_prio' % prefix
            if self.hasGraph(graph_name):
                if not stats.has_key(prefix):
                    stats[prefix] = proc_info.getProcStatStatus(is_thread)
                for (fname, stat_key) in (
                    ('high', 'high'),
                    ('low', 'low'),
                    ('norm', 'norm'),
                    ('locked', 'locked_in_mem')):
                    self.setGraphVal(graph_name, fname, 
                                     stats[prefix]['prio'].get(stat_key))
997627
def workflow(ctx, client):
    """"""
    if ctx.invoked_subcommand is None:
        from renku.models.refs import LinkReference

        names = defaultdict(list)
        for ref in LinkReference.iter_items(client, common_path='workflows'):
            names[ref.reference.name].append(ref.name)

        for path in client.workflow_path.glob('*.cwl'):
            click.echo(
                '{path}: {names}'.format(
                    path=path.name,
                    names=', '.join(
                        click.style(_deref(name), fg='green')
                        for name in names[path.name]
                    ),
                )
            )
811238
def _get_sentinel_val(v):
    
    out = workflow.get_base_id(v["id"])
    if workflow.is_cwl_record(v):
        out += ":%s" % ";".join([x["name"] for x in _get_record_fields(v)])
    return out