Merges calcs into a new calc called DATASET
        NB: this is the PUBLIC method
        @returns DATASET, error
187052
def merge(self, session, checksums, title):
        '''
        
        '''
        calc = Output(calcset=checksums)

        cur_depth = 0

        for nested_depth, grid_item, download_size in session.query(model.Calculation.nested_depth, model.Grid.info, model.Metadata.download_size).filter(model.Calculation.checksum == model.Grid.checksum, model.Grid.checksum == model.Metadata.checksum, model.Calculation.checksum.in_(checksums)).all():

            if nested_depth > cur_depth: cur_depth = nested_depth

            grid_item = json.loads(grid_item)

            for entity in self.hierarchy:

                topic = grid_item.get(entity['source'])
                if not topic:
                    continue

                if not isinstance(topic, list):
                    topic = [ topic ]

                calc.info[ entity['source'] ] = list(set( calc.info.get(entity['source'], []) + topic ))

            calc.download_size += download_size

        if not calc.download_size:
            return None, 'Wrong parameters provided!'

        calc._nested_depth = cur_depth + 1

        calc.info['standard'] = title

        # generate fake checksum
        calc._checksum = calc.get_collective_checksum()

        return calc, None
1125385
def insertDataset(self, dataset):
        
        try:
            models.Dataset.create(
                id=dataset.getId(),
                name=dataset.getLocalId(),
                description=dataset.getDescription(),
                attributes=json.dumps(dataset.getAttributes()))
        except Exception:
            raise exceptions.DuplicateNameException(
                dataset.getLocalId())
1219386
public static BatchCSVRecord fromDataSet(DataSet dataSet) {
        BatchCSVRecord batchCSVRecord = new BatchCSVRecord();
        for (int i = 0; i < dataSet.numExamples(); i++) {
            batchCSVRecord.add(SingleCSVRecord.fromRow(dataSet.get(i)));
        }

        return batchCSVRecord;
    }
838098
public LCMSDataSubset merge(LCMSDataSubset other) {
    LCMSDataSubset merged = new LCMSDataSubset();

    Set<Integer> msLvlsThis = getMsLvls();
    Set<Integer> msLvlsThat = other.getMsLvls();
    // only merge if both are not null, otherwise null signifies the whole
    // run, so we can keep it null in the merged version
    if (msLvlsThis != null && msLvlsThat != null) {
      HashSet<Integer> mergedMsLvls = new HashSet<>(msLvlsThis);
      mergedMsLvls.addAll(msLvlsThat);
      merged.setMsLvls(mergedMsLvls);
    }

    // merge mz ranges
    List<DoubleRange> mzRangesThis = getMzRanges();
    List<DoubleRange> mzRangesThat = other.getMzRanges();
    if (mzRangesThis != null && mzRangesThat != null) {
      ArrayList<DoubleRange> mergedMzRanges = new ArrayList<>(mzRangesThis);
      mergedMzRanges.addAll(mzRangesThat);
      merged.setMzRanges(mergedMzRanges);
    }

    // compare scan number ranges
    Integer scanNumLoThis = getScanNumLo();
    Integer scanNumLoThat = other.getScanNumLo();
    if (scanNumLoThis != null && scanNumLoThat != null) {
      merged.setScanNumLo(Math.min(scanNumLoThis, scanNumLoThat));
    }
    Integer scanNumHiThis = getScanNumHi();
    Integer scanNumHiThat = other.getScanNumHi();
    if (scanNumHiThis != null && scanNumHiThat != null) {
      merged.setScanNumHi(Math.max(scanNumHiThis, scanNumHiThat));
    }

    return merged;
  }
573103
def _calc_concat_over(datasets, dim, data_vars, coords):
    
    # Return values
    concat_over = set()
    equals = {}

    if dim in datasets[0]:
        concat_over.add(dim)
    for ds in datasets:
        concat_over.update(k for k, v in ds.variables.items()
                           if dim in v.dims)

    def process_subset_opt(opt, subset):
        if isinstance(opt, str):
            if opt == 'different':
                # all nonindexes that are not the same in each dataset
                for k in getattr(datasets[0], subset):
                    if k not in concat_over:
                        # Compare the variable of all datasets vs. the one
                        # of the first dataset. Perform the minimum amount of
                        # loads in order to avoid multiple loads from disk
                        # while keeping the RAM footprint low.
                        v_lhs = datasets[0].variables[k].load()
                        # We'll need to know later on if variables are equal.
                        computed = []
                        for ds_rhs in datasets[1:]:
                            v_rhs = ds_rhs.variables[k].compute()
                            computed.append(v_rhs)
                            if not v_lhs.equals(v_rhs):
                                concat_over.add(k)
                                equals[k] = False
                                # computed variables are not to be re-computed
                                # again in the future
                                for ds, v in zip(datasets[1:], computed):
                                    ds.variables[k].data = v.data
                                break
                        else:
                            equals[k] = True

            elif opt == 'all':
                concat_over.update(set(getattr(datasets[0], subset)) -
                                   set(datasets[0].dims))
            elif opt == 'minimal':
                pass
            else:
                raise ValueError("unexpected value for %s: %s" % (subset, opt))
        else:
            invalid_vars = [k for k in opt
                            if k not in getattr(datasets[0], subset)]
            if invalid_vars:
                if subset == 'coords':
                    raise ValueError(
                        'some variables in coords are not coordinates on '
                        'the first dataset: %s' % (invalid_vars,))
                else:
                    raise ValueError(
                        'some variables in data_vars are not data variables '
                        'on the first dataset: %s' % (invalid_vars,))
            concat_over.update(opt)

    process_subset_opt(data_vars, 'data_vars')
    process_subset_opt(coords, 'coords')
    return concat_over, equals
192068
def copy_dataset_files(self, ds, incver=False, cb=None, **kwargs):
        
        from ambry.orm import File

        tables = [File]

        return self._copy_dataset_copy(ds, tables, incver, cb, **kwargs)
871218
def merge_dfs(dfs, resample=None, do_mean=False, do_sum=False, do_min=False, do_max=False):
    
    if len(dfs) == 0:
        raise Exception("No dataframes provided")
    df = dfs.values()[0]
    name = dfs.keys()[0]
    df.columns = map(lambda x: name+"_"+x if not x.startswith(name) else x, df.columns)
    if resample is not None:
        df = df.resample(resample)
        if do_mean: df = df.mean()
        elif do_sum: df = df.sum()
        elif do_min: df = df.min()
        elif do_max: df = df.max()
        else: df = df.mean()
    if len(dfs) > 1:
        for name, newdf in dfs.items()[1:]:
            if resample is not None:
                newdf = newdf.resample(resample)
                if do_mean: newdf = newdf.mean()
                elif do_sum: newdf = newdf.sum()
                elif do_min: newdf = newdf.min()
                elif do_max: newdf = newdf.max()
                else: newdf = newdf.mean()
            newdf.columns = map(lambda x: name+"_"+x if not x.startswith(name) else x, newdf.columns)
            df = df.merge(newdf, left_index=True, right_index=True, how='outer')
    return df
1184470
def apply_dataset_vfunc(
    func,
    *args,
    signature,
    join='inner',
    dataset_join='exact',
    fill_value=_NO_FILL_VALUE,
    exclude_dims=frozenset(),
    keep_attrs=False
):
    
    from .dataset import Dataset
    first_obj = args[0]  # we'll copy attrs from this in case keep_attrs=True

    if (dataset_join not in _JOINS_WITHOUT_FILL_VALUES and
            fill_value is _NO_FILL_VALUE):
        raise TypeError('to apply an operation to datasets with different '
                        'data variables with apply_ufunc, you must supply the '
                        'dataset_fill_value argument.')

    if len(args) > 1:
        args = deep_align(args, join=join, copy=False, exclude=exclude_dims,
                          raise_on_invalid=False)

    list_of_coords = build_output_coords(args, signature, exclude_dims)
    args = [getattr(arg, 'data_vars', arg) for arg in args]

    result_vars = apply_dict_of_variables_vfunc(
        func, *args, signature=signature, join=dataset_join,
        fill_value=fill_value)

    if signature.num_outputs > 1:
        out = tuple(_fast_dataset(*args)
                    for args in zip(result_vars, list_of_coords))
    else:
        coord_vars, = list_of_coords
        out = _fast_dataset(result_vars, coord_vars)

    if keep_attrs and isinstance(first_obj, Dataset):
        if isinstance(out, tuple):
            out = tuple(ds._copy_attrs_from(first_obj) for ds in out)
        else:
            out._copy_attrs_from(first_obj)
    return out
81853
def main(dataset1, dataset2, target):
    
    d1 = read_raw(dataset1)
    d2 = read_raw(dataset2)
    merged = merge(d1, d2)
    with open(target, 'wb') as f:
        pickle.dump(merged, f, protocol=pickle.HIGHEST_PROTOCOL)
1139987
@Override
    public DataSet next() {
        counter.incrementAndGet();

        DataSet ds = new DataSet(baseFeatures, baseLabels);

        return ds;
    }
192285
def addDataset(self, dataset):
        
        self._datasets.append(dataset)
        self._dataChanged = True
        self._addDatasetAction(dataset)
518590
public function prepend(Dataset $dataset) : Dataset
    {
        return self::quick(array_merge($dataset->samples(), $this->samples));
    }
528579
public static int numExamplesTrain(Set dataSet) {
        switch (dataSet) {
            case COMPLETE:
                return NUM_COMPLETE_TRAIN;
            case MERGE:
                return NUM_MERGE_TRAIN;
            case BALANCED:
                return NUM_BALANCED_TRAIN;
            case LETTERS:
                return NUM_LETTERS_TRAIN;
            case DIGITS:
                return NUM_DIGITS_TRAIN;
            case MNIST:
                return NUM_MNIST_TRAIN;
            default:
                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
        }
    }
919684
def GetMerger(self, cls):
    
    for merger in self._mergers:
      if isinstance(merger, cls):
        return merger
    raise LookupError('No matching DataSetMerger found')
291115
def table(self, datatype=None, **kwargs):
        
        if config.future_deprecations:
            self.param.warning("The table method is deprecated and should no "
                               "longer be used. If using a HoloMap use "
                               "HoloMap.collapse() instead to return a Dataset.")

        from .data.interface import Interface
        from ..element.tabular import Table
        new_data = [(key, value.table(datatype=datatype, **kwargs))
                    for key, value in self.data.items()]
        tables = self.clone(new_data)
        return Interface.concatenate(tables, new_type=Table)
1189618
def merge(self, parent=None):
        
        # Check parent, append source files
        if parent is None:
            parent = self.parent
        if parent is None:
            return
        self.sources = parent.sources + self.sources
        # Merge child data with parent data
        data = copy.deepcopy(parent.data)
        for key, value in sorted(self.data.items()):
            # Handle attribute adding
            if key.endswith('+'):
                key = key.rstrip('+')
                if key in data:
                    # Use dict.update() for merging dictionaries
                    if type(data[key]) == type(value) == dict:
                        data[key].update(value)
                        continue
                    try:
                        value = data[key] + value
                    except TypeError as error:
                        raise utils.MergeError(
                            "MergeError: Key '{0}' in {1} ({2}).".format(
                                key, self.name, str(error)))
            # And finally update the value
            data[key] = value
        self.data = data
545542
public static DataSet copyOf(DataSet data) {
    if (data == null) {
      throw new InvalidOperationException("Null data set");
    }
    DataSet r = new DataSet(data.getSource());
    r.getRows().addAll(data.getRows());
    return r;
  }