Convert world data object (or list of data objects) into outputs for CWL ingestion.
197263
def _convert_to_cwl_json(data, fnargs, input_files):
    
    out = {}
    for outvar in _get_output_cwl_keys(fnargs):
        keys = []
        for key in outvar.split("__"):
            try:
                key = int(key)
            except ValueError:
                pass
            keys.append(key)
        if isinstance(data, dict):
            out[outvar] = _to_cwl(tz.get_in(keys, data), input_files)
        else:
            out[outvar] = [_to_cwl(tz.get_in(keys, x), input_files) for x in data]
    return out
93219
def ls(ctx, list_formatted):
    """"""
    session = create_session(ctx.obj['AWS_PROFILE_NAME'])

    rds = session.client('rds')
    instances = rds.describe_db_instances()
    out = format_output(instances['DBInstances'], list_formatted)
    click.echo('\n'.join(out))
1344180
def consume(self, cwd=None):
        
        first_pass = Grammar.overall.parseString(self.string)
        lowered = { key.lower(): val for key, val in first_pass.iteritems() }

        self.commands = ['\n'.join(self._get('commands', lowered))]
        self.job_options = self._get('job_options', lowered)
        self.global_options = self._get('options', lowered)

        self.files = self._get('files', lowered)
        self.paths = self._get('paths', lowered)

        self.files = self._parse(self.files, Grammar.file, True)
        self.paths = self._parse(self.paths, Grammar.path, True)
        self.job_options = self._parse(self.job_options, Grammar.line)

        try:
            command_lines = self._parse(self.commands, Grammar.command_lines)[0]
        except IndexError:
            raise ValueError('Did you write any commands?')

        self.commands = []
        for command_line in command_lines:
            comments, command = command_line
            self.commands.append([comments.asList(),
                self._parse([''.join(command)], Grammar.command)])

        self.job_options = [opt.asList() for opt in self.job_options]

        self.paths = ctf.get_paths(self.paths)
        self.files = ctf.get_files(self.files)

        self.paths.reverse()
        self.files.reverse()
        self.commands.reverse()

        return ctf.get_command_templates(self.commands, self.files[:],
            self.paths[:], self.job_options)
1084273
def global_items(self):
        """"""
        for (k, v) in self.sql('global_dump'):
            yield (self.unpack(k), self.unpack(v))
370941
def _ingest_list(self, input_list, schema_list, path_to_root):

        '''
            
        '''

        valid_list = []

    # construct max list size
        max_size = None
        rules_path_to_root = re.sub('\[\d+\]', '[0]', path_to_root)
        if 'max_size' in self.keyMap[rules_path_to_root].keys():
            if not self.keyMap[rules_path_to_root]['max_size']:
                return valid_list
            else:
                max_size = self.keyMap[rules_path_to_root]['max_size']

    # iterate over items in input list
        if input_list:
            rules_index = self._datatype_classes.index(schema_list[0].__class__)
            rules_type = self._datatype_names[rules_index]
            for i in range(len(input_list)):
                item_path = '%s[%s]' % (path_to_root, i)
                value_match = False
                try:
                    item_index = self._datatype_classes.index(input_list[i].__class__)
                    item_type = self._datatype_names[item_index]
                    if item_type == rules_type:
                        value_match = True
                except:
                    value_match = False
                if value_match:
                    try:
                        if item_type == 'boolean':
                            valid_list.append(self._validate_boolean(input_list[i], item_path))
                        elif item_type == 'number':
                            valid_list.append(self._validate_number(input_list[i], item_path))
                        elif item_type == 'string':
                            valid_list.append(self._validate_string(input_list[i], item_path))
                        elif item_type == 'map':
                            valid_list.append(self._ingest_dict(input_list[i], schema_list[0], item_path))
                        elif item_type == 'list':
                            valid_list.append(self._ingest_list(input_list[i], schema_list[0], item_path))
                    except:
                        pass
                if isinstance(max_size, int):
                    if len(valid_list) == max_size:
                        return valid_list

        return valid_list
984750
def _wf_to_dict(wf, records):
    
    inputs, outputs, records = _get_wf_inout(wf, records)
    out = {"name": _id_to_name(_clean_id(wf.tool["id"])), "inputs": inputs,
           "outputs": outputs, "steps": [], "subworkflows": [],
           "requirements": []}
    for step in wf.steps:
        is_subworkflow = isinstance(step.embedded_tool, cwltool.workflow.Workflow)
        inputs, outputs, remapped, prescatter = _get_step_inout(step)
        inputs, scatter = _organize_step_scatter(step, inputs, remapped)
        if is_subworkflow:
            wf_def, records = _wf_to_dict(step.embedded_tool, records)
            out["subworkflows"].append({"id": "%s.%s" % (wf_def["name"], wf_def["name"]), "definition": wf_def,
                                        "inputs": inputs, "outputs": outputs, "scatter": scatter,
                                        "prescatter": prescatter})
        else:
            task_def, records = _tool_to_dict(step.embedded_tool, records, remapped)
            out["steps"].append({"task_id": task_def["name"], "task_definition": task_def,
                                 "inputs": inputs, "outputs": outputs, "scatter": scatter,
                                 "prescatter": prescatter})
    return out, records
1180692
def convert_to_pysal(data):
    
    w = pysal.lat2W(len(data[0]), len(data))
    data = np.array(data)
    data = np.reshape(data, (len(data)*len(data[0]), 1))
    return w, data
327626
def to_table_data(self):
        

        self._validate_source_data()
        self._loader.inc_table_count()

        yield TableData(
            self._make_table_name(),
            ["key", "value"],
            [record for record in self._buffer.items()],
            dp_extractor=self._loader.dp_extractor,
            type_hints=self._extract_type_hints(),
        )
586249
def chunk_data(data, chunkSize):
    '''
        
    '''
    chunkSize = int(chunkSize)
    return [data[i : i + chunkSize] for i in range(0, len(data), chunkSize)]
1333140
def _decompose(net, wv_map, mems, block_out):
    

    def arg(x, i):
        # return the mapped wire vector for argument x, wire number i
        return wv_map[(net.args[x], i)]

    def destlen():
        # return iterator over length of the destination in bits
        return range(len(net.dests[0]))

    def assign_dest(i, v):
        # assign v to the wiremap for dest[0], wire i
        wv_map[(net.dests[0], i)] <<= v

    one_var_ops = {
        'w': lambda w: w,
        '~': lambda w: ~w,
    }
    c_two_var_ops = {
        '&': lambda l, r: l & r,
        '|': lambda l, r: l | r,
        '^': lambda l, r: l ^ r,
        'n': lambda l, r: l.nand(r),
    }

    if net.op in one_var_ops:
        for i in destlen():
            assign_dest(i, one_var_ops[net.op](arg(0, i)))
    elif net.op in c_two_var_ops:
        for i in destlen():
            assign_dest(i, c_two_var_ops[net.op](arg(0, i), arg(1, i)))
    elif net.op == 's':
        for i in destlen():
            selected_bit = arg(0, net.op_param[i])
            assign_dest(i, selected_bit)
    elif net.op == 'c':
        arg_wirelist = []
        # generate list of wires for vectors being concatenated
        for arg_vector in net.args:
            arg_vector_as_list = [wv_map[(arg_vector, i)] for i in range(len(arg_vector))]
            arg_wirelist = arg_vector_as_list + arg_wirelist
        for i in destlen():
            assign_dest(i, arg_wirelist[i])
    elif net.op == 'r':
        for i in destlen():
            args = (arg(0, i),)
            dests = (wv_map[(net.dests[0], i)],)
            new_net = LogicNet('r', None, args=args, dests=dests)
            block_out.add_net(new_net)
    elif net.op == 'm':
        arg0list = [arg(0, i) for i in range(len(net.args[0]))]
        addr = concat_list(arg0list)
        new_mem = _get_new_block_mem_instance(net.op_param, mems, block_out)[1]
        data = as_wires(new_mem[addr])
        for i in destlen():
            assign_dest(i, data[i])
    elif net.op == '@':
        addrlist = [arg(0, i) for i in range(len(net.args[0]))]
        addr = concat_list(addrlist)
        datalist = [arg(1, i) for i in range(len(net.args[1]))]
        data = concat_list(datalist)
        enable = arg(2, 0)
        new_mem = _get_new_block_mem_instance(net.op_param, mems, block_out)[1]
        new_mem[addr] <<= MemBlock.EnabledWrite(data=data, enable=enable)
    else:
        raise PyrtlInternalError('Unable to synthesize the following net '
                                 'due to unimplemented op :\n%s' % str(net))
    return
1296409
def mapped(cls, key='id', **kwargs):
    """"""
    kwargs.setdefault('metadata', {})
    kwargs['metadata']['jsonldPredicate'] = {'mapSubject': key}
    kwargs.setdefault('default', attr.Factory(list))

    def converter(value):
        """Convert mapping to a list of instances."""
        if isinstance(value, dict):
            result = []
            for k, v in iteritems(value):
                if not hasattr(cls, 'from_cwl'):
                    vv = dict(v)
                    vv[key] = k
                else:
                    vv = attr.evolve(cls.from_cwl(v), **{key: k})
                result.append(vv)
        else:
            result = value

        def fix_keys(data):
            """Fix names of keys."""
            for a in fields(cls):
                a_name = a.name.rstrip('_')
                if a_name in data:
                    yield a.name, data[a_name]

        return [
            cls(**{kk: vv
                   for kk, vv in fix_keys(v)}) if not isinstance(v, cls) else v
            for v in result
        ]

    kwargs['converter'] = converter
    return attr.ib(**kwargs)
1021789
def record_to_objects(self):
        """"""
        from ambry.orm import SourceTable

        bsfile = self.record

        failures = set()

        # Clear out all of the columns from existing tables. We don't clear out the
        # tables, since they may be referenced by sources

        for row in bsfile.dict_row_reader:
            st = self._dataset.source_table(row['table'])

            if st:
                st.columns[:] = []

        self._dataset.commit()

        for row in bsfile.dict_row_reader:
            st = self._dataset.source_table(row['table'])

            if not st:
                st = self._dataset.new_source_table(row['table'])
                # table_number += 1

            if 'datatype' not in row:
                row['datatype'] = 'unknown'

            del row['table']

            st.add_column(**row)  # Create or update

        if failures:
            raise ConfigurationError('Failed to load source schema, missing sources: {} '.format(failures))

        self._dataset.commit()
1013061
def results_to_table(self):
        """"""
        # Get Labels
        baseline_labels = self.nearest_targets['origin_id'].values
        mcf_labels = self.results['model_labels']['labels'].values

        # Create the outcomes
        outcome = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(self.results['model_labels']['lng'].values, self.results['model_labels']['lat'].values)],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lng': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(baseline_labels)['lng'].values,
                'origin_lat': self.origins.reindex(baseline_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': baseline_labels
            }
        )

        outcomes2 = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(
                        self.results['model_labels']['lng'].values,
                        self.results['model_labels']['lat'].values
                    )
                ],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lat': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(mcf_labels)['lng'].values,
                'origin_lat': self.origins.reindex(mcf_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': mcf_labels
            },
            index=self.results['model_labels'].target_id
        )
        now = datetime.datetime.now()
        out_table = 'mincostflow_{}'.format(now.strftime("%Y_%m_%d_%H_%M_%S"))
        logging.info('Writing output to {}'.format(out_table))
        self.context.write(outcomes2.reset_index(drop=True), out_table)
        logging.info('Table {} written to CARTO'.format(out_table))
        return out_table
694466
def _nested_cwl_record(xs, want_attrs, input_files):
    
    if isinstance(xs, (list, tuple)):
        return [_nested_cwl_record(x, want_attrs, input_files) for x in xs]
    else:
        assert isinstance(xs, dict), pprint.pformat(xs)
        return _collapse_to_cwl_record_single(xs, want_attrs, input_files)
436256
def raw(self, output):
        """"""
        if isinstance(output, (list, tuple)):
            output = '\n'.join(map(str, output))
        self.outfile.write(str(output))
1059496
def stream_object_data(self, ref):
		""""""
		cmd = self.__get_persistent_cmd("cat_file_all", "cat_file", batch=True)
		hexsha, typename, size = self.__get_object_header(cmd, ref)
		return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout))
16321
def _serialize_item(item):
    """"""
    # Recursively serialize lists, tuples, and dicts.
    if isinstance(item, (list, tuple)):
        return [_serialize_item(subitem) for subitem in item]
    elif isinstance(item, dict):
        return dict([(key, _serialize_item(value))
                     for (key, value) in iteritems(item)])

    # Serialize strings.
    elif isinstance(item, string_types):
        # Replace glSomething by something (needed for WebGL commands).
        if item.startswith('gl'):
            return re.sub(r'^gl([A-Z])', lambda m: m.group(1).lower(), item)
        else:
            return item

    # Process NumPy arrays that are not buffers (typically, uniform values).
    elif isinstance(item, np.ndarray):
        return _serialize_item(item.ravel().tolist())

    # Serialize numbers.
    else:
        try:
            return np.asscalar(item)
        except Exception:
            return item