Persist dataset state of a given dataset identified by the dataset URN.
199580
private void persistDatasetState(String datasetUrn, JobState.DatasetState datasetState)
      throws IOException {
    log.info("Persisting dataset state for dataset " + datasetUrn);
    this.jobContext.getDatasetStateStore().persistDatasetState(datasetUrn, datasetState);
  }
437837
def continuousSetsGenerator(self, request):
        
        dataset = self.getDataRepository().getDataset(request.dataset_id)
        return self._topLevelObjectGenerator(
            request, dataset.getNumContinuousSets(),
            dataset.getContinuousSetByIndex)
251860
public static JavaRDD<DataSet> fromContinuousLabeledPoint(JavaRDD<LabeledPoint> data, boolean preCache) {
        if (preCache && !data.getStorageLevel().useMemory()) {
            data.cache();
        }
        return data.map(new Function<LabeledPoint, DataSet>() {
            @Override
            public DataSet call(LabeledPoint lp) {
                return convertToDataset(lp);
            }
        });
    }
583292
def dataset_status_cli(self, dataset, dataset_opt=None):
        
        dataset = dataset or dataset_opt
        return self.dataset_status(dataset)
1039914
def save( self ):
        
        dataSet = self.dataSet()
        if ( not dataSet ):
            return True
        
        # lookup widgets based on the data set information
        for widget in self.findChildren(QWidget):
            prop = unwrapVariant(widget.property('dataName'))
            if prop is None:
                continue
            
            # update the data for the dataset
            value, success = projexui.widgetValue(widget)
            if not success:
                continue
                
            dataSet.setValue(prop, value)
        
        return self.plugin().save()
58212
def reset( self ):
        
        dataSet = self.dataSet()
        if ( not dataSet ):
            dataSet = XScheme()
        
        dataSet.reset()
1018316
def (self):
        """"""

        archive_data = {"packets": self.recv_msgs,
                        "dataset": self.dataset_name,
                        "num_packets": len(self.recv_msgs),
                        "created": rnow()}
        self.write_to_file(archive_data,
                           self.archive_file)
1316746
def data_available(dataset_name=None):
    """"""
    try:
        from itertools import zip_longest
    except ImportError:
        from itertools import izip_longest as zip_longest
    dr = data_resources[dataset_name]
    zip_urls = (dr['files'], )
    if 'save_names' in dr: zip_urls += (dr['save_names'], )
    else: zip_urls += ([],)

    for file_list, save_list in zip_longest(*zip_urls, fillvalue=[]):
        for f, s in zip_longest(file_list, save_list, fillvalue=None):
            if s is not None: f=s # If there is a save_name given, use that one
            if not os.path.exists(os.path.join(data_path, dataset_name, f)):
                return False
    return True
1286915
def addDataset(self, dataset):
        
        id_ = dataset.getId()
        self._datasetIdMap[id_] = dataset
        self._datasetNameMap[dataset.getLocalId()] = dataset
        self._datasetIds.append(id_)
571676
def storedata(filename=None):
    """"""
    if filename is None:
        filename = 'credolib_state.pickle'
    ns = get_ipython().user_ns
    with open(filename, 'wb') as f:
        d = {}
        for var in ['_headers', '_loaders', '_data1d', '_data2d', '_data1dunited',
                    'allsamplenames', '_headers_sample', 'badfsns', '_rowavg',
                    'saveto_dir',
                    'badfsns_datcmp', 'auximages_dir', 'subtractedsamplenames', 'outputpath', 'saveto_dir_rel',
                    'auximages_dir_rel', 'crd_prefix']:
            try:
                d[var] = ns[var]
            except KeyError:
                warnings.warn('Skipping storage of unavailable variable "%s"' % var)
        pickle.dump(d, f)
1320440
def open_dataset(self, service):
        """"""

        if not self.dataset:
            path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
            self.dataset = netCDF4.Dataset(path, 'r')
        return self.dataset
222073
def get_dataset(self, dataset):
        
        # If the dataset is present, no need to download anything.
        success = True
        dataset_path = self.base_dataset_path + dataset
        if not isdir(dataset_path):

            # Try 5 times to download. The download page is unreliable, so we need a few tries.
            was_error = False
            for iteration in range(5):

                # Guard against trying again if successful
                if iteration == 0 or was_error is True:
                    zip_path = dataset_path + ".zip"

                    # Download zip files if they're not there
                    if not isfile(zip_path):
                        try:
                            with DLProgress(unit='B', unit_scale=True, miniters=1, desc=dataset) as pbar:
                                urlretrieve(self.datasets[dataset]["url"], zip_path, pbar.hook)
                        except Exception as ex:
                            print("Error downloading %s: %s" % (dataset, ex))
                            was_error = True

                    # Unzip the data files
                    if not isdir(dataset_path):
                        try:
                            with zipfile.ZipFile(zip_path) as zip_archive:
                                zip_archive.extractall(path=dataset_path)
                                zip_archive.close()
                        except Exception as ex:
                            print("Error unzipping %s: %s" % (zip_path, ex))
                            # Usually the error is caused by a bad zip file.
                            # Delete it so the program will try to download it again.
                            try:
                                remove(zip_path)
                            except FileNotFoundError:
                                pass
                            was_error = True

            if was_error:
                print("\nThis recognizer is trained by the CASIA handwriting database.")
                print("If the download doesn't work, you can get the files at %s" % self.datasets[dataset]["url"])
                print("If you have download problems, "
                      "wget may be effective at downloading because of download resuming.")
                success = False

        return success
974887
private void finalizeDatasetStateBeforeCommit(JobState.DatasetState datasetState) {
    for (TaskState taskState : datasetState.getTaskStates()) {
      if (taskState.getWorkingState() != WorkUnitState.WorkingState.SUCCESSFUL
          && this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) {
        // The dataset state is set to FAILED if any task failed and COMMIT_ON_FULL_SUCCESS is used
        datasetState.setState(JobState.RunningState.FAILED);
        datasetState.incrementJobFailures();
        return;
      }
    }

    datasetState.setState(JobState.RunningState.SUCCESSFUL);
    datasetState.setNoJobFailure();
  }
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)
1293423
def updateDataset(self):
        """"""
        self.__updateDataItem()
        self.__countNbRecords()
        self.__columns = self.__parseColumns()
162309
def add_dataset(data_type, val, unit_id=None, metadata={}, name="", user_id=None, flush=False):
    

    d = Dataset()

    d.type  = data_type
    d.value = val
    d.set_metadata(metadata)

    d.unit_id  = unit_id
    d.name  = name
    d.created_by = user_id
    d.hash  = d.set_hash()

    try:
        existing_dataset = db.DBSession.query(Dataset).filter(Dataset.hash==d.hash).one()
        if existing_dataset.check_user(user_id):
            d = existing_dataset
        else:
            d.set_metadata({'created_at': datetime.datetime.now()})
            d.set_hash()
            db.DBSession.add(d)
    except NoResultFound:
        db.DBSession.add(d)

    if flush == True:
        db.DBSession.flush()
    return d
1219386
public static BatchCSVRecord fromDataSet(DataSet dataSet) {
        BatchCSVRecord batchCSVRecord = new BatchCSVRecord();
        for (int i = 0; i < dataSet.numExamples(); i++) {
            batchCSVRecord.add(SingleCSVRecord.fromRow(dataSet.get(i)));
        }

        return batchCSVRecord;
    }