Creates and caches dataset info object. Subsequent invocations will
return same instance.

@see IIM#DS(int, int)
@param dataSet
dataset record number
@return dataset info instace
199055
public DataSetInfo create(int dataSet) throws InvalidDataSetException {
		DataSetInfo info = dataSets.get(createKey(dataSet));
		if (info == null) {
			int recordNumber = (dataSet >> 8) & 0xFF;
			int dataSetNumber = dataSet & 0xFF;
			throw new UnsupportedDataSetException(recordNumber + ":" + dataSetNumber);
			// info = super.create(dataSet);
		}
		return info;
	}
312912
def copy_dataset(self, ds, incver=False, cb=None, **kwargs):
        
        from ambry.orm import Table, Column, Partition, File, ColumnStat, Code, \
            DataSource, SourceTable, SourceColumn

        tables = [Table, Column, Partition, File, ColumnStat, Code, SourceTable, SourceColumn, DataSource]

        return self._copy_dataset_copy(ds, tables, incver, cb, **kwargs)
694983
@Override
    public DataSet copy() {
        DataSet ret = new DataSet(getFeatures().dup(), getLabels().dup());
        if (getLabelsMaskArray() != null)
            ret.setLabelsMaskArray(getLabelsMaskArray().dup());
        if (getFeaturesMaskArray() != null)
            ret.setFeaturesMaskArray(getFeaturesMaskArray().dup());
        ret.setColumnNames(getColumnNames());
        ret.setLabelNames(getLabelNames());
        return ret;
    }
269544
public static DatasetId of(String project, String dataset) {
    return new DatasetId(checkNotNull(project), checkNotNull(dataset));
  }
858598
def create(cls, train_ds:Dataset, valid_ds:Dataset, test_ds:Optional[Dataset]=None, path:PathOrStr='.', bs:int=64,
               val_bs:int=None, num_workers:int=defaults.cpus, dl_tfms:Optional[Collection[Callable]]=None,
               device:torch.device=None, collate_fn:Callable=data_collate, no_check:bool=False, **dl_kwargs)->'DataBunch':
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        dls = [DataLoader(d, b, shuffle=s, drop_last=s, num_workers=num_workers, **dl_kwargs) for d,b,s in
               zip(datasets, (bs,val_bs,val_bs,val_bs), (True,False,False,False)) if d is not None]
        return cls(*dls, path=path, device=device, dl_tfms=dl_tfms, collate_fn=collate_fn, no_check=no_check)
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
199580
private void persistDatasetState(String datasetUrn, JobState.DatasetState datasetState)
      throws IOException {
    log.info("Persisting dataset state for dataset " + datasetUrn);
    this.jobContext.getDatasetStateStore().persistDatasetState(datasetUrn, datasetState);
  }
916077
def _inner_dataset_template(cls, dataset):
        
        from . import Dataset
        vdims = dataset.vdims if getattr(dataset, 'level', None) is None else []
        return Dataset(dataset.data[0], datatype=cls.subtypes,
                       kdims=dataset.kdims, vdims=vdims)
1145162
def load(self, ds, verbose=False):
        
        try:
            self.fetchone('select ID from dataset')
        except sqlite3.OperationalError:
            self.create(force=True)
        self.unload(ds)
        dataset = ds.cldf.wl
        tables, ref_tables = schema(dataset)

        # update the DB schema:
        for t in tables:
            if self._create_table_if_not_exists(t):
                continue
            db_cols = self.tables[t.name]
            for col in t.columns:
                if col.name not in db_cols:
                    with self.connection() as conn:
                        conn.execute(
                            "ALTER TABLE {0} ADD COLUMN `{1.name}` {1.db_type}".format(
                                t.name, col))
                else:
                    if db_cols[col.name] != col.db_type:
                        raise ValueError(
                            'column {0}:{1} {2} redefined with new type {3}'.format(
                                t.name, col.name, db_cols[col.name], col.db_type))

        for t in ref_tables.values():
            self._create_table_if_not_exists(t)

        self.update_schema()

        # then load the data:
        with self.connection() as db:
            db.execute('PRAGMA foreign_keys = ON;')
            insert(
                db,
                'dataset',
                'ID,name,version,metadata_json',
                (
                    ds.id,
                    '{0}'.format(dataset),
                    git_hash(ds.dir),
                    dumps(dataset.metadata_dict)))
            insert(
                db,
                'datasetmeta',
                'dataset_ID,key,value',
                *[(ds.id, k, '{0}'.format(v)) for k, v in dataset.properties.items()])

            # load sources:
            rows = []
            for src in dataset.sources.items():
                values = [ds.id, src.id, src.genre] + [src.get(k) for k in BIBTEX_FIELDS]
                values.append(
                    dumps({k: v for k, v in src.items() if k not in BIBTEX_FIELDS}))
                rows.append(tuple(values))
            insert(
                db,
                'SourceTable',
                ['dataset_ID', 'ID', 'bibtex_type'] + BIBTEX_FIELDS + ['extra'],
                *rows)

            # For regular tables, we extract and keep references to sources.
            refs = defaultdict(list)

            for t in tables:
                # We want to lookup columns by the name used in the CLDF dataset.
                cols = {col.cldf_name: col for col in t.columns}
                # But we also want to look up primary keys by the database column name.
                cols_by_name = {col.name: col for col in t.columns}

                ref_table = ref_tables.get(t.name)
                rows, keys = [], []
                try:
                    for row in dataset[t.name]:
                        keys, values = ['dataset_ID'], [ds.id]
                        for k, v in row.items():
                            if ref_table and k == ref_table.consumes:
                                col = cols_by_name[t.primary_key]
                                refs[ref_table.name].append((row[col.cldf_name], v))
                            else:
                                col = cols[k]
                                if isinstance(v, list):
                                    v = (col.separator or ';').join(
                                        nfilter(col.convert(vv) for vv in v))
                                else:
                                    v = col.convert(v)
                                keys.append("`{0}`".format(col.name))
                                values.append(v)
                        keys, values = self.update_row(t.name, keys, values)
                        rows.append(tuple(values))
                    insert(db, t.name, keys, *rows, **{'verbose': verbose})
                except FileNotFoundError:
                    if t.name != 'CognateTable':  # An empty CognateTable is allowed.
                        raise  # pragma: no cover

            # Now insert the references, i.e. the associations with sources:
            for tname, items in refs.items():
                rows = []
                for oid, sources in items:
                    for source in sources:
                        sid, context = Sources.parse(source)
                        rows.append([ds.id, oid, sid, context])
                oid_col = '{0}_ID'.format(tname.replace('Source', ''))
                insert(db, tname, ['dataset_ID', oid_col, 'Source_ID', 'Context'], *rows)
            db.commit()
131804
public CompactionParserResult parse (FileSystemDataset dataset) {

    CompactionParserResult result = new CompactionParserResult();
    result.srcBaseDir = getSrcBaseDir (state);
    result.srcSubDir  = getSrcSubDir  (state);
    result.dstBaseDir = getDstBaseDir (state);
    result.dstSubDir  = getDstSubDir  (state);

    parseTimeAndDatasetName(dataset, result);

    result.dstAbsoluteDir = Joiner.on("/").join (result.dstBaseDir,
                                                  result.datasetName,
                                                  result.dstSubDir,
                                                  result.timeString);
    return result;
  }
528579
public static int numExamplesTrain(Set dataSet) {
        switch (dataSet) {
            case COMPLETE:
                return NUM_COMPLETE_TRAIN;
            case MERGE:
                return NUM_MERGE_TRAIN;
            case BALANCED:
                return NUM_BALANCED_TRAIN;
            case LETTERS:
                return NUM_LETTERS_TRAIN;
            case DIGITS:
                return NUM_DIGITS_TRAIN;
            case MNIST:
                return NUM_MNIST_TRAIN;
            default:
                throw new UnsupportedOperationException("Unknown Set: " + dataSet);
        }
    }
1021536
public FormInputHandler dataSet(final DataSet theDataSet) {
		checkState(dataSet == null, "Cannot specify a DataSet because a direct value is already set.");

		Fields fields = new Fields(this);
		fields.dataSet = theDataSet;
		return new FormInputHandler(fields);
	}
1363314
def delete_dataset(self, dataset, delete_nonempty=True):
        

        if not self.dataset_exists(dataset):
            return

        self.client.datasets().delete(projectId=dataset.project_id,
                                      datasetId=dataset.dataset_id,
                                      deleteContents=delete_nonempty).execute()
192068
def copy_dataset_files(self, ds, incver=False, cb=None, **kwargs):
        
        from ambry.orm import File

        tables = [File]

        return self._copy_dataset_copy(ds, tables, incver, cb, **kwargs)
137484
public DSet newDSet()
    {
        if ((getCurrentDatabase() == null))
        {
            throw new DatabaseClosedException("Database is NULL, cannot create a DSet with a null database.");
        }
        return (DSet) DSetFactory.singleton.createCollectionOrMap(getCurrentPBKey());
    }
259781
public static void prepareDataSource(DataSource ds) throws SQLException {
    Connection con = null;
    String sql;
    try {
      con = ds.getConnection();
      con.setAutoCommit(false);

      DatabaseMetaData meta = con.getMetaData();
      ResultSet rs = meta.getTables(null, null, null, new String[]{"TABLE"});
      int tableNameColumn = rs.findColumn("TABLE_NAME");
      int count = 0;
      while (rs.next()) {
        String tableName = rs.getString(tableNameColumn);
        if (tableName.equalsIgnoreCase("CachedClasses")) {
          count++;
        }
      }

      if (count == 0) {
        String blobType = DbUtil.getTypeName(Types.BLOB, con);
        String nameType = DbUtil.getTypeName(Types.VARCHAR, 1024, con);
        String md5Type = DbUtil.getTypeName(Types.BINARY, 16, con);

        sql =  "CREATE TABLE CachedClasses ( \n" +
            "  Name " + nameType + " NOT NULL, \n" +
            "  MD5 " + md5Type + " NOT NULL, \n" +
            "  Definition " + blobType + " NOT NULL, \n" +
            "  PRIMARY KEY (Name, MD5) \n" +
            ")";
        DbUtil.update(ds, sql);
        con.commit();
      }

      con.setAutoCommit(true);
    } catch (SQLException e) {
      DbUtil.rollback(con);
      throw e;
    } finally {
      DbUtil.close(con);
    }
  }
943417
public static List<String> getLabels(Set dataSet) {
        char[] c = getLabelsArray(dataSet);
        List<String> l = new ArrayList<>(c.length);
        for (char c2 : c) {
            l.add(String.valueOf(c2));
        }
        return l;
    }