Apply metadata to help formatting the output:

        - conv_funcs: convert data before column alignments
        - deco_funcs: decorate data after column alignments
480605
def apply_meta(self, arr, funcs):
        
        if arr is None:
            logging.error("unable to convert data: emtpy section")
            return None

        tmp = np.copy(arr)
        for col in arr.dtype.names:
            for row in self._get_row_hdrs():
                meta = self._get_meta(row, col)
                for mk, mv in sorted(meta.iteritems()):
                    if mk in funcs.iterkeys():
                        tmp[col][self.irt[row]] = \
                            funcs[mk](tmp[col][self.irt[row]], mv)
        return tmp
482126
def objify(self, doc, columns=None):
        
        cols = columns or doc[METADATA][COLUMNS]
        data = {}

        for col in cols:
            # if there is missing data in a chunk, we can default to NaN
            # and pandas will autofill the missing values to the correct length
            if col not in doc[METADATA][LENGTHS]:
                d = [np.nan]
            else:
                d = decompress(doc[DATA][doc[METADATA][LENGTHS][col][0]: doc[METADATA][LENGTHS][col][1] + 1])
                # d is ready-only but that's not an issue since DataFrame will copy the data anyway.
                d = np.frombuffer(d, doc[METADATA][DTYPE][col])

                if MASK in doc[METADATA] and col in doc[METADATA][MASK]:
                    mask_data = decompress(doc[METADATA][MASK][col])
                    mask = np.frombuffer(mask_data, 'bool')
                    d = ma.masked_array(d, mask)
            data[col] = d

        # Copy into
        return pd.DataFrame(data, columns=cols, copy=True)[cols]
1231387
def _wrap_deprecated_function(func, message):
    
    def _(col):
        warnings.warn(message, DeprecationWarning)
        return func(col)
    return functools.wraps(func)(_)
888037
def _collate_metadata(self):
        
        import pandas as pd

        DEFAULT_FIELDS = None
        metadata = []

        for c in self._classifications:
            m = c.sample.metadata

            if DEFAULT_FIELDS is None:
                DEFAULT_FIELDS = list(m._resource._schema["properties"].keys())
                DEFAULT_FIELDS.remove("$uri")
                DEFAULT_FIELDS.remove("sample")

            metadatum = {f: getattr(m, f) for f in DEFAULT_FIELDS}
            metadatum["classification_id"] = c.id
            metadatum["sample_id"] = m.sample.id
            metadatum["metadata_id"] = m.id
            metadatum["created_at"] = m.sample.created_at
            metadatum["filename"] = c.sample.filename

            metadatum.update(m.custom)
            metadata.append(metadatum)

        if metadata:
            metadata = pd.DataFrame(metadata).set_index("classification_id")
        else:
            metadata = pd.DataFrame(
                columns=["classification_id", "sample_id", "metadata_id", "created_at"]
            )

        self._cached["metadata"] = metadata
417214
def _mask_cov_func(self, *args):
        
        # Have to do it in two cases to get the 1d unwrapped properly:
        if self.num_dim == 1:
            return self.cov_func(args[0], args[1], *self.params)
        else:
            return self.cov_func(args[:self.num_dim], args[self.num_dim:], *self.params)
515246
def make_funcs(dataset, setdir, store):
    """"""
    return {
        'cat': lambda *lists: [x for lst in lists for x in lst],
        'comments': lambda: None,
        'detail_route': detail_route,
        'format': lambda fmt, *args: fmt.format(*args),
        'get': partial(getnode, dataset, setdir, store),
        'join': lambda sep, *args: sep.join([x for x in args if x]),
        'len': len,
        'link': (lambda href, title, target=None:
                 {'href': href or "",
                  'title': title or "",
                  'target': '_blank' if target is None else target}),
        'list': lambda *x: filter(None, list(x)),
        'max': max,
        'min': min,
        'status': lambda: ['#STATUS#'],
        'sum': sum,
        'tags': lambda: ['#TAGS#'],
        'trace': print_trace,
    }
308182
def format(input, **params):
    
    PARAM_FORMAT_STRING = 'format.string'
    PARAM_FORMAT_INPUT = 'format.input'
    PARAM_RESULT_FIELD = 'result.field'
    IN_DESC_FIELD = 'field'
    IN_DESC_TYPE = 'type'

    format_string = params.get(PARAM_FORMAT_STRING)
    format_inputs = params.get(PARAM_FORMAT_INPUT)
    result_field = params.get(PARAM_RESULT_FIELD)
    for row in input:
        row_input = []
        for desc in format_inputs:
            row_input.append(Converter.convert(row[desc[IN_DESC_FIELD]], desc[IN_DESC_TYPE]))
        row[result_field] = format_string.format(*row_input)
    return input
1339862
def make_preprocessing_fn(output_dir, features, keep_target):
  
  def preprocessing_fn(inputs):
    """Preprocessing function.

    Args:
      inputs: dictionary of raw input tensors

    Returns:
      A dictionary of transformed tensors
    """
    stats = json.loads(
      file_io.read_file_to_string(
          os.path.join(output_dir, STATS_FILE)).decode())

    result = {}
    for name, transform in six.iteritems(features):
      transform_name = transform['transform']
      source_column = transform['source_column']

      if transform_name == KEY_TRANSFORM:
        transform_name = 'identity'
      elif transform_name == TARGET_TRANSFORM:
        if not keep_target:
          continue
        if file_io.file_exists(os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column)):
          transform_name = 'one_hot'
        else:
          transform_name = 'identity'

      if transform_name == 'identity':
        result[name] = inputs[source_column]
      elif transform_name == 'scale':
        result[name] = _scale(
            inputs[name],
            min_x_value=stats['column_stats'][source_column]['min'],
            max_x_value=stats['column_stats'][source_column]['max'],
            output_min=transform.get('value', 1) * (-1),
            output_max=transform.get('value', 1))
      elif transform_name in [ONE_HOT_TRANSFORM, EMBEDDING_TRANSFROM, MULTI_HOT_TRANSFORM,
                              TFIDF_TRANSFORM, BOW_TRANSFORM]:
        vocab, ex_count = read_vocab_file(
            os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column))

        if transform_name == TFIDF_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _tfidf(
              x=ids,
              reduced_term_freq=ex_count + [0],
              vocab_size=len(vocab) + 1,
              corpus_size=stats['num_examples'])

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == BOW_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _bag_of_words(x=ids)

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == MULTI_HOT_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          result[name] = _string_to_int(tokens, vocab)
        else:
          # ONE_HOT_TRANSFORM: making a dense vector is done at training
          # EMBEDDING_TRANSFROM: embedding vectors have to be done at training
          result[name] = _string_to_int(inputs[source_column], vocab)
      elif transform_name == IMAGE_TRANSFORM:
        make_image_to_vec_fn = _make_image_to_vec_tito(
            name, checkpoint=transform.get('checkpoint', None))
        result[name] = make_image_to_vec_fn(inputs[source_column])
      else:
        raise ValueError('unknown transform %s' % transform_name)
    return result

  return preprocessing_fn
84064
def read_with_columns(func):
    
    def wrapper(*args, **kwargs):
        # parse columns argument
        columns = kwargs.pop("columns", None)

        # read table
        tab = func(*args, **kwargs)

        # filter on columns
        if columns is None:
            return tab
        return tab[columns]

    return _safe_wraps(wrapper, func)
1307679
def custom_function(func, input_files, output_file):
    
    from .modules import utils

    return {
        'dependencies_fn': utils.no_dependencies,
        'compiler_fn': func,
        'input': input_files,
        'output': output_file,
        'kwargs': {},
    }
440212
def _output_format(self, out, fmt_j=None, fmt_p=None):
        
        if self.output_format == 'pandas':
            if fmt_p is not None:
                return fmt_p(out)
            else:
                return self._convert_output(out)
        if fmt_j:
            return fmt_j(out)
        return out
427843
def format_doc(fun):
    """"""
    SEPARATOR = '============================='
    func = cvloop.functions.__dict__[fun]

    doc_lines = ['{}'.format(l).strip() for l in func.__doc__.split('\n')]
    if hasattr(func, '__init__'):
        doc_lines.append(SEPARATOR)
        doc_lines += ['{}'.format(l).strip() for l in
                      func.__init__.__doc__.split('\n')]

    mod_lines = []
    argblock = False
    returnblock = False
    for line in doc_lines:
        if line == SEPARATOR:
            mod_lines.append('\n#### `{}.__init__(...)`:\n\n'.format(fun))
        elif 'Args:' in line:
            argblock = True
            if GENERATE_ARGS:
                mod_lines.append('**{}**\n'.format(line))
        elif 'Returns:' in line:
            returnblock = True
            mod_lines.append('\n**{}**'.format(line))
        elif not argblock and not returnblock:
            mod_lines.append('{}\n'.format(line))
        elif argblock and not returnblock and ':' in line:
            if GENERATE_ARGS:
                mod_lines.append('- *{}:* {}\n'.format(
                    *line.split(':')))
        elif returnblock:
            mod_lines.append(line)
        else:
            mod_lines.append('{}\n'.format(line))
    return mod_lines
1184470
def apply_dataset_vfunc(
    func,
    *args,
    signature,
    join='inner',
    dataset_join='exact',
    fill_value=_NO_FILL_VALUE,
    exclude_dims=frozenset(),
    keep_attrs=False
):
    
    from .dataset import Dataset
    first_obj = args[0]  # we'll copy attrs from this in case keep_attrs=True

    if (dataset_join not in _JOINS_WITHOUT_FILL_VALUES and
            fill_value is _NO_FILL_VALUE):
        raise TypeError('to apply an operation to datasets with different '
                        'data variables with apply_ufunc, you must supply the '
                        'dataset_fill_value argument.')

    if len(args) > 1:
        args = deep_align(args, join=join, copy=False, exclude=exclude_dims,
                          raise_on_invalid=False)

    list_of_coords = build_output_coords(args, signature, exclude_dims)
    args = [getattr(arg, 'data_vars', arg) for arg in args]

    result_vars = apply_dict_of_variables_vfunc(
        func, *args, signature=signature, join=dataset_join,
        fill_value=fill_value)

    if signature.num_outputs > 1:
        out = tuple(_fast_dataset(*args)
                    for args in zip(result_vars, list_of_coords))
    else:
        coord_vars, = list_of_coords
        out = _fast_dataset(result_vars, coord_vars)

    if keep_attrs and isinstance(first_obj, Dataset):
        if isinstance(out, tuple):
            out = tuple(ds._copy_attrs_from(first_obj) for ds in out)
        else:
            out._copy_attrs_from(first_obj)
    return out
443140
def cols_(df,col_list,func,*args,**kwargs):
    
    return _pd.DataFrame({col_name:func(df,col_name,*args,**kwargs) for col_name in col_list})
526857
def col_mod(df,col_name,func,*args,**kwargs):
    
    backup = df[col_name].copy()
    try:
        return_val = func(df,col_name,*args,**kwargs)
        if return_val is not None:
            set_col(df,col_name,return_val)
    except:
        df[col_name] = backup
949207
def __parse(self, function_meta):
        
        self._func = get_mapping_function(
            function_meta["func_name"],
            self.functions_mapping
        )
        self.func_name = self._func.__name__
        self._args = prepare_lazy_data(
            function_meta.get("args", []),
            self.functions_mapping,
            self.check_variables_set
        )
        self._kwargs = prepare_lazy_data(
            function_meta.get("kwargs", {}),
            self.functions_mapping,
            self.check_variables_set
        )

        if self.func_name == "load_csv_file":
            if len(self._args) != 1 or self._kwargs:
                raise exceptions.ParamsError("P() should only pass in one argument!")
            self._args = [self._args[0]]
        elif self.func_name == "get_os_environ":
            if len(self._args) != 1 or self._kwargs:
                raise exceptions.ParamsError("ENV() should only pass in one argument!")
            self._args = [self._args[0]]
474818
def _format_trace(trace):
    
    lines = []
    for fname, lineno, func, src, _ in trace:
        if src:
            for line in src:
                lines.append('    '+line.strip()+'\n')
        lines.append('  %s:%4d in %s\n' % (fname, lineno, func))
    return ''.join(lines)