Combines whatever datasets that can be combined,
    and save the bigger dataset to a given location.
546687
def combine_and_save(add_path_list, out_path):
    

    add_path_list = list(add_path_list)
    # first one!
    first_ds_path = add_path_list[0]
    print('Starting with {}'.format(first_ds_path))
    combined = MLDataset(first_ds_path)
    for ds_path in add_path_list[1:]:
        try:
            combined = combined + MLDataset(ds_path)
        except:
            print('      Failed to add {}'.format(ds_path))
            traceback.print_exc()
        else:
            print('Successfully added {}'.format(ds_path))

    combined.save(out_path)

    return
497233
def to_xarray_dataset(self, datasets=None):
        
        if datasets is not None:
            datasets = [self[ds] for ds in datasets]
        else:
            datasets = [self.datasets.get(ds) for ds in self.wishlist]
            datasets = [ds for ds in datasets if ds is not None]

        ds_dict = {i.attrs['name']: i.rename(i.attrs['name']) for i in datasets if i.attrs.get('area') is not None}
        mdata = combine_metadata(*tuple(i.attrs for i in datasets))
        if mdata.get('area') is None or not isinstance(mdata['area'], SwathDefinition):
            # either don't know what the area is or we have an AreaDefinition
            ds = xr.merge(ds_dict.values())
        else:
            # we have a swath definition and should use lon/lat values
            lons, lats = mdata['area'].get_lonlats()
            if not isinstance(lons, DataArray):
                lons = DataArray(lons, dims=('y', 'x'))
                lats = DataArray(lats, dims=('y', 'x'))
            # ds_dict['longitude'] = lons
            # ds_dict['latitude'] = lats
            ds = xr.Dataset(ds_dict, coords={"latitude": (["y", "x"], lats),
                                             "longitude": (["y", "x"], lons)})

        ds.attrs = mdata
        return ds
140414
def merge_resultsets(result1, result2)
      (result1.keys | result2.keys).each_with_object({}) do |filename, merged|
        file1 = result1[filename]
        file2 = result2[filename]
        merged[filename] = merge_file_coverage(file1, file2)
      end
    end
650241
def attach(domain, filename):
    '''
    
    '''
    log.info('Attaching datasets for domain %s', domain)
    result = actions.attach(domain, filename)
    log.info('Attached %s datasets to %s', result.success, domain)
790574
def _save_multi(data, file_name, sep=";"):
    """"""
    logger.debug("saving multi")
    with open(file_name, "w", newline='') as f:
        logger.debug(f"{file_name} opened")
        writer = csv.writer(f, delimiter=sep)
        try:
            writer.writerows(itertools.zip_longest(*data))
        except Exception as e:
            logger.info(f"Exception encountered in batch._save_multi: {e}")
            raise ExportFailed
        logger.debug("wrote rows using itertools in _save_multi")
573103
def _calc_concat_over(datasets, dim, data_vars, coords):
    
    # Return values
    concat_over = set()
    equals = {}

    if dim in datasets[0]:
        concat_over.add(dim)
    for ds in datasets:
        concat_over.update(k for k, v in ds.variables.items()
                           if dim in v.dims)

    def process_subset_opt(opt, subset):
        if isinstance(opt, str):
            if opt == 'different':
                # all nonindexes that are not the same in each dataset
                for k in getattr(datasets[0], subset):
                    if k not in concat_over:
                        # Compare the variable of all datasets vs. the one
                        # of the first dataset. Perform the minimum amount of
                        # loads in order to avoid multiple loads from disk
                        # while keeping the RAM footprint low.
                        v_lhs = datasets[0].variables[k].load()
                        # We'll need to know later on if variables are equal.
                        computed = []
                        for ds_rhs in datasets[1:]:
                            v_rhs = ds_rhs.variables[k].compute()
                            computed.append(v_rhs)
                            if not v_lhs.equals(v_rhs):
                                concat_over.add(k)
                                equals[k] = False
                                # computed variables are not to be re-computed
                                # again in the future
                                for ds, v in zip(datasets[1:], computed):
                                    ds.variables[k].data = v.data
                                break
                        else:
                            equals[k] = True

            elif opt == 'all':
                concat_over.update(set(getattr(datasets[0], subset)) -
                                   set(datasets[0].dims))
            elif opt == 'minimal':
                pass
            else:
                raise ValueError("unexpected value for %s: %s" % (subset, opt))
        else:
            invalid_vars = [k for k in opt
                            if k not in getattr(datasets[0], subset)]
            if invalid_vars:
                if subset == 'coords':
                    raise ValueError(
                        'some variables in coords are not coordinates on '
                        'the first dataset: %s' % (invalid_vars,))
                else:
                    raise ValueError(
                        'some variables in data_vars are not data variables '
                        'on the first dataset: %s' % (invalid_vars,))
            concat_over.update(opt)

    process_subset_opt(data_vars, 'data_vars')
    process_subset_opt(coords, 'coords')
    return concat_over, equals
843911
def _combine_sets(self, sets, final_set):
        
        if self._has_sortedsets:
            self.cls.get_connection().zinterstore(final_set, list(sets))
        else:
            final_set = super(ExtendedCollectionManager, self)._combine_sets(sets, final_set)
        return final_set
713734
def saveDataSet( settings, key, dataSet ):
    
    for datakey, value in dataSet.items():
        datatype  = type(value).__name__
        
        if ( datatype in _dataValueTypes ):
            datavalue = _dataValueTypes[datatype][0](value)
        else:
            datavalue = value
        
        settings.setValue('%s/%s/type' % (key, datakey), wrapVariant(datatype))
        settings.setValue('%s/%s/value' % (key, datakey), wrapVariant(datavalue))
343834
def bundle(self, ref, capture_exceptions=False):
        """"""
        from ..orm.exc import NotFoundError

        if isinstance(ref, Dataset):
            ds = ref
        else:
            try:
                ds = self._db.dataset(ref)
            except NotFoundError:
                ds = None

        if not ds:
            try:
                p = self.partition(ref)
                ds = p._bundle.dataset
            except NotFoundError:
                ds = None

        if not ds:
            raise NotFoundError('Failed to find dataset for ref: {}'.format(ref))

        b = Bundle(ds, self)
        b.capture_exceptions = capture_exceptions

        return b
1130509
def push(description, datasets, wait, verbose):
    """"""
    # Loading config
    config = _get_config()
    file_patterns = config["contents"]
    if not isinstance(file_patterns, type([])):
        # put it into an array so we can iterate it, if it isn't already an array
        file_patterns = [file_patterns]
    if datasets:
        datasets_string = datasets.split(',')
    else:
        datasets_string = config.get("datasets", '')


    # Getting all file names/globs -- making sure we get CONFIG_FILE
    files = {CONFIG_FILE}  # use a set so we don't get duplicates
    for pattern in file_patterns:
        files.update(glob.glob(pattern))
    if not files:
        print "No files could be found? Check your contents section in `CONFIG_FILE`!"
        exit(-1)

    if verbose:
        print ""
        print "gatherered files:"
        for path in files:
            print "\t", path
        print ""

    # Zipping
    _print("zipping...")

    if not os.path.exists("brains_history"):
        os.mkdir("brains_history")

    zip_path = 'brains_history/%s.zip' % str(datetime.datetime.now())
    with ZipFile(zip_path, 'w') as zip_file:
        for path in files:
            zip_file.write(path)
    cprint("done", 'green')

    # Sending to server
    with open(zip_path, 'rb') as zip_file:
        _print("sending to server...")
        try:
            response = requests.post(
                URL_SUBMIT,
                files={
                    "zip_file": zip_file,
                },
                data={
                    "name": config["name"],
                    "description": description or '',
                    "languages": config["languages"],
                    "datasets": datasets_string,
                    "wait": wait,
                },
                stream=wait  # if we're waiting for response then stream
            )

            if response.status_code == 200:
                cprint("done", 'green')

                if wait:
                    _print("\nOutput: ")
                    cprint(" " * 72, 'green', attrs=('underline',))

                    chunk_buffer = ""
                    # read in 1 chunk at a time for carriage return detection
                    for chunk in response.iter_content(chunk_size=1):
                        chunk_buffer += chunk

                        if chunk == '\r':
                            # We hit the end of a message!
                            try:
                                data = json.loads(chunk_buffer)
                                if "stdout" not in data or "stderr" not in data:
                                    print "dis one"
                                    continue
                            except (ValueError,):
                                continue

                            if data["stdout"]:
                                # Get rid of termination string, if it's there
                                data["stdout"] = data["stdout"].replace("-%-%-%-%-END BRAIN SEQUENCE-%-%-%-%-", "")
                                _print(data["stdout"])
                            if data["stderr"]:
                                _print(colored(data["stderr"], 'red'))

                            # Clear buffer after reading message
                            chunk_buffer = ""
            else:
                cprint(response.json()["error"], 'red')
        except requests.exceptions.ConnectionError:
            cprint("failed to connect to server!", 'red')
            exit(-2)
717587
def _merge_metadata(samples):
    """"""
    samples = list(utils.flatten(samples))
    out_dir = dd.get_work_dir(samples[0])
    logger.info("summarize metadata")
    out_file = os.path.join(out_dir, "metadata.csv")
    sample_metrics = collections.defaultdict(dict)
    for s in samples:
        m = tz.get_in(['metadata'], s)
        if isinstance(m, six.string_types):
            m = json.loads(m)
        if m:
            for me in list(m.keys()):
                if isinstance(m[me], list) or isinstance(m[me], dict) or isinstance(m[me], tuple):
                    m.pop(me, None)
            sample_metrics[dd.get_sample_name(s)].update(m)
    pd.DataFrame(sample_metrics).transpose().to_csv(out_file)
    return out_file
1096164
def merge(*maps):
    """"""
    copies = map(deepcopy, maps)
    return reduce(lambda acc, val: acc.update(val) or acc, copies)
1316746
def data_available(dataset_name=None):
    """"""
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    dr = data_resources[dataset_name]
    zip_urls = (dr['files'], )
    if 'save_names' in dr: zip_urls += (dr['save_names'], )
    else: zip_urls += ([],)

    for file_list, save_list in zip_longest(*zip_urls, fillvalue=[]):
        for f, s in zip_longest(file_list, save_list, fillvalue=None):
            if s is not None: f=s # If there is a save_name given, use that one
            if not os.path.exists(os.path.join(data_path, dataset_name, f)):
                return False
    return True
786683
def dataset_merge_method(dataset, other, overwrite_vars, compat, join):
    """"""

    # we are locked into supporting overwrite_vars for the Dataset.merge
    # method due for backwards compatibility
    # TODO: consider deprecating it?

    if isinstance(overwrite_vars, str):
        overwrite_vars = set([overwrite_vars])
    overwrite_vars = set(overwrite_vars)

    if not overwrite_vars:
        objs = [dataset, other]
        priority_arg = None
    elif overwrite_vars == set(other):
        objs = [dataset, other]
        priority_arg = 1
    else:
        other_overwrite = OrderedDict()
        other_no_overwrite = OrderedDict()
        for k, v in other.items():
            if k in overwrite_vars:
                other_overwrite[k] = v
            else:
                other_no_overwrite[k] = v
        objs = [dataset, other_no_overwrite, other_overwrite]
        priority_arg = 2

    return merge_core(objs, compat, join, priority_arg=priority_arg)
611176
public void merge(DataStore other) {
        if (!this.hasSameColumns(other)) {
            throw new IllegalArgumentException("DataStore must have the same columns to merge");
        }

        this.rows.putAll(other.rows);
    }
321706
def addDataset(self):
        
        self._openRepo()
        dataset = datasets.Dataset(self._args.datasetName)
        dataset.setDescription(self._args.description)
        dataset.setAttributes(json.loads(self._args.attributes))
        self._updateRepo(self._repo.insertDataset, dataset)
1039914
def save( self ):
        
        dataSet = self.dataSet()
        if ( not dataSet ):
            return True
        
        # lookup widgets based on the data set information
        for widget in self.findChildren(QWidget):
            prop = unwrapVariant(widget.property('dataName'))
            if prop is None:
                continue
            
            # update the data for the dataset
            value, success = projexui.widgetValue(widget)
            if not success:
                continue
                
            dataSet.setValue(prop, value)
        
        return self.plugin().save()