This function merges datasets into one set.
968518
def merge(self, datasets=None, separate_datasets=False):
        """"""
        self.logger.info("merging")
        if separate_datasets:
            warnings.warn("The option seperate_datasets=True is"
                          "not implemented yet. Performing merging, but"
                          "neglecting the option.")
        else:
            if datasets is None:
                datasets = list(range(len(self.datasets)))
            first = True
            for dataset_number in datasets:
                if first:
                    dataset = self.datasets[dataset_number]
                    first = False
                else:
                    dataset = self._append(dataset, self.datasets[dataset_number])
                    for raw_data_file, file_size in zip(self.datasets[dataset_number].raw_data_files,
                                                        self.datasets[dataset_number].raw_data_files_length):
                        dataset.raw_data_files.append(raw_data_file)
                        dataset.raw_data_files_length.append(file_size)
            self.datasets = [dataset]
            self.number_of_datasets = 1
        return self
1184470
def apply_dataset_vfunc(
    func,
    *args,
    signature,
    join='inner',
    dataset_join='exact',
    fill_value=_NO_FILL_VALUE,
    exclude_dims=frozenset(),
    keep_attrs=False
):
    
    from .dataset import Dataset
    first_obj = args[0]  # we'll copy attrs from this in case keep_attrs=True

    if (dataset_join not in _JOINS_WITHOUT_FILL_VALUES and
            fill_value is _NO_FILL_VALUE):
        raise TypeError('to apply an operation to datasets with different '
                        'data variables with apply_ufunc, you must supply the '
                        'dataset_fill_value argument.')

    if len(args) > 1:
        args = deep_align(args, join=join, copy=False, exclude=exclude_dims,
                          raise_on_invalid=False)

    list_of_coords = build_output_coords(args, signature, exclude_dims)
    args = [getattr(arg, 'data_vars', arg) for arg in args]

    result_vars = apply_dict_of_variables_vfunc(
        func, *args, signature=signature, join=dataset_join,
        fill_value=fill_value)

    if signature.num_outputs > 1:
        out = tuple(_fast_dataset(*args)
                    for args in zip(result_vars, list_of_coords))
    else:
        coord_vars, = list_of_coords
        out = _fast_dataset(result_vars, coord_vars)

    if keep_attrs and isinstance(first_obj, Dataset):
        if isinstance(out, tuple):
            out = tuple(ds._copy_attrs_from(first_obj) for ds in out)
        else:
            out._copy_attrs_from(first_obj)
    return out
1350336
def merge(*ds):
    
    if not ds:
        raise ValueError("Must provide at least one dict to merge().")
    out = {}
    for d in ds:
        out.update(d)
    return out
192068
def copy_dataset_files(self, ds, incver=False, cb=None, **kwargs):
        
        from ambry.orm import File

        tables = [File]

        return self._copy_dataset_copy(ds, tables, incver, cb, **kwargs)
640878
def merge_sources(self, datas):
        
        datas = [data for data in datas if data is not None]

        if len(datas) == 0:
            raise ValueError("Data missing")

        if len(datas) == 1:
            return datas[0]

        if isinstance(datas[0], list):
            if len([x for x in datas if not isinstance(x, list)]) > 0:
                raise TypeError("Unable to merge: List expected")
            base = []
            for x in datas:
                base = base + x
            return base

        if isinstance(datas[0], dict):
            if len([x for x in datas if not isinstance(x, dict)]) > 0:
                raise TypeError("Unable to merge: Dictionnary expected")
            result = {}
            for element in datas:
                for key in element:
                    if key in result:
                        result[key] = self.merge_sources([result[key], element[key]])
                    else:
                        result[key] = element[key]
            return result

        if len([x for x in datas if isinstance(x, (dict, list))]) > 0:
            raise TypeError("Unable to merge: List not expected")

        raise ValueError("Unable to merge: Conflict")
314005
def marvcli_undiscard(datasets):
    """"""
    create_app()

    setids = parse_setids(datasets, discarded=True)
    dataset = Dataset.__table__
    stmt = dataset.update()\
                  .where(dataset.c.setid.in_(setids))\
                  .values(discarded=False)
    db.session.execute(stmt)
    db.session.commit()
271506
function mergeData (to, from) {
	  var key, toVal, fromVal;
	  for (key in from) {
	    toVal = to[key];
	    fromVal = from[key];
	    if (!hasOwn(to, key)) {
	      set(to, key, fromVal);
	    } else if (isObject(toVal) && isObject(fromVal)) {
	      mergeData(toVal, fromVal);
	    }
	  }
	  return to
	}
105619
def _bg_combine(self, bgs):
        """"""
        out = np.ones(self.h5["raw"].shape, dtype=float)
        # bg is an h5py.DataSet
        for bg in bgs:
            out *= bg[:]
        return out
866451
def _merge(dts):
    
    df = pd.concat(dts)

    ma = df.pivot(index='isomir', columns='sample', values='counts')
    ma_mirna = ma
    ma = ma.fillna(0)
    ma_mirna['mirna'] = [m.split(":")[0] for m in ma.index.values]
    ma_mirna = ma_mirna.groupby(['mirna']).sum()
    ma_mirna = ma_mirna.fillna(0)
    return ma, ma_mirna
93901
def _merge_results(self, results):
        """"""
        self.results['tests'] += results['tests']

        for key, value in results['summary'].items():
            self.results['summary'][key] += value
1203617
def merge(cls, tables, fillna=False):
        """"""
        cols = set(itertools.chain(*[table.dtype.descr for table in tables]))

        tables_to_merge = []
        for table in tables:
            missing_cols = cols - set(table.dtype.descr)

            if missing_cols:
                if fillna:
                    n = len(table)
                    n_cols = len(missing_cols)
                    col_names = []
                    for col_name, col_dtype in missing_cols:
                        if 'f' not in col_dtype:
                            raise ValueError(
                                "Cannot create NaNs for non-float"
                                " type column '{}'".format(col_name)
                            )
                        col_names.append(col_name)

                    table = table.append_columns(
                        col_names, np.full((n_cols, n), np.nan)
                    )
                else:
                    raise ValueError(
                        "Table columns do not match. Use fill_na=True"
                        " if you want to append missing values with NaNs"
                    )
            tables_to_merge.append(table)

        first_table = tables_to_merge[0]

        merged_table = sum(tables_to_merge[1:], first_table)

        merged_table.h5loc = first_table.h5loc
        merged_table.h5singleton = first_table.h5singleton
        merged_table.split_h5 = first_table.split_h5
        merged_table.name = first_table.name

        return merged_table
207941
def shuffle_and_batch_data(dataset,
                           target_names,
                           features_info,
                           training,
                           num_devices,
                           shuffle_buffer_size=1024,
                           preprocess_fun=no_preprocess):
  """"""
  def append_targets(example):
    """Append targets to the example dictionary. Needed for Keras."""
    if len(target_names) == 1:
      return (example, example[target_names[0]])
    targets = {}
    for name in target_names:
      targets[name] = example[name]
    return (example, targets)
  dataset = dataset.map(append_targets)
  if training:
    dataset = dataset.repeat()
    # Skip a random fraction at the beginning of the stream.  The skip is
    # essential for synchronous highly-parallel training to avoid multiple
    # replicas reading the same data in lock-step.
    dataset = dataset.skip(random.randint(0, _MAX_SKIP_EXAMPLES))
  dataset = preprocess_fun(dataset, training)
  shapes = {k: features_info[k].shape for k in features_info}
  shapes = (shapes, shapes[target_names[0]])
  dataset = dataset.shuffle(shuffle_buffer_size)
  dataset = batch_fun(dataset, training, shapes, target_names, num_devices)
  return dataset.prefetch(2)
1146242
def merge(self, ds, inplace=False, axis=1):
        
        if not isinstance(ds, Dataset):
            raise ValueError('Expected `Dataset`, got %s.' % ds)

        X_train = concat(ds.X_train, self.X_train, axis=axis)
        y_train = concat(ds.y_train, self.y_train, axis=axis)

        if ds.X_test is not None:
            X_test = concat(ds.X_test, self.X_test, axis=axis)
        else:
            X_test = None

        if ds.y_test is not None:
            y_test = concat(ds.y_test, self.y_test, axis=axis)
        else:
            y_test = None

        if inplace:
            self._X_train = X_train
            self._y_train = y_train

            if X_test is not None:
                self._X_test = X_test

            if y_test is not None:
                self._y_test = y_test

            return None

        return Dataset(X_train, y_train, X_test, y_test)
1312643
def split(cls, dataset, start, end, datatype, **kwargs):
        
        objs = []
        if datatype is None:
            for d in dataset.data[start: end]:
                objs.append(dataset.clone(d, datatype=cls.subtypes))
            return objs
        elif not dataset.data:
            return objs
        ds = cls._inner_dataset_template(dataset)
        for d in dataset.data:
            ds.data = d
            if datatype == 'array':
                obj = ds.array(**kwargs)
            elif datatype == 'dataframe':
                obj = ds.dframe(**kwargs)
            elif datatype == 'columns':
                if ds.interface.datatype == 'dictionary':
                    obj = dict(ds.data)
                else:
                    obj = ds.columns(**kwargs)
            else:
                raise ValueError("%s datatype not support" % datatype)
            objs.append(obj)
        return objs
717587
def _merge_metadata(samples):
    """"""
    samples = list(utils.flatten(samples))
    out_dir = dd.get_work_dir(samples[0])
    logger.info("summarize metadata")
    out_file = os.path.join(out_dir, "metadata.csv")
    sample_metrics = collections.defaultdict(dict)
    for s in samples:
        m = tz.get_in(['metadata'], s)
        if isinstance(m, six.string_types):
            m = json.loads(m)
        if m:
            for me in list(m.keys()):
                if isinstance(m[me], list) or isinstance(m[me], dict) or isinstance(m[me], tuple):
                    m.pop(me, None)
            sample_metrics[dd.get_sample_name(s)].update(m)
    pd.DataFrame(sample_metrics).transpose().to_csv(out_file)
    return out_file
160124
def from_dataset(cls, dataset, **kwargs):
        """"""
        m = cls(dataset.dim_x, dataset.dim_y, **kwargs)
        m.dataset = dataset
        return m
1009048
def _combine_sets(self, sets, final_set):
        
        self.cls.get_connection().sinterstore(final_set, list(sets))
        return final_set