Returns all primary dataset types if dataset or primary_ds_type are not passed.
972662
def listPrimaryDSTypes(self, primary_ds_type="", dataset=""):
        
        conn = self.dbi.connection()
        try:
            result = self.primdstypeList.execute(conn, primary_ds_type, dataset)
            if conn: conn.close()
            return result
        finally:
            if conn:
                conn.close()
824631
def listDatasetParents(self, dataset=""):
        
        if( dataset == "" ):
            dbsExceptionHandler("dbsException-invalid-input", "DBSDataset/listDatasetParents. Child Dataset name is required.")
        conn = self.dbi.connection()
        try:
            result = self.datasetparentlist.execute(conn, dataset)
            return result
        finally:
            if conn:
                conn.close()
1100802
def _check_has_primary(sds):
    
    for s in sds.values():
        if s.server_type == SERVER_TYPE.RSPrimary:
            return TOPOLOGY_TYPE.ReplicaSetWithPrimary
    else:
        return TOPOLOGY_TYPE.ReplicaSetNoPrimary
1185168
def std_check_in(dataset, name, allowed_vals):
    
    if not hasattr(dataset, name):
        return 0

    ret_val = 1
    if getattr(dataset, name) in allowed_vals:
        ret_val += 1

    return ret_val
1038750
def get_all_resources(datasets):
        # type: (List['Dataset']) -> List[hdx.data.resource.Resource]
        
        resources = []
        for dataset in datasets:
            for resource in dataset.get_resources():
                resources.append(resource)
        return resources
720419
def get_dsn(d):
    

    try:
        return d["dataSetName"]
    except Exception as e:
        logger_misc.warn("get_dsn: Exception: No datasetname found, unable to continue: {}".format(e))
        exit(1)
701833
def root_dataset(self):
        """"""
        ds = self.dataset(ROOT_CONFIG_NAME_V)
        ds._database = self
        return ds
343896
public Set<Type> getDataModelTypes()
        throws CacheReloadException
    {
        final Set<Type> ret = new HashSet<>();
        for (final Long id : this.dataModelTypes) {
            ret.add(Type.get(id));
        }
        return Collections.unmodifiableSet(ret);
    }
77876
def column_types(self):
        
        column_types = {}
        for c in self.sqla_columns:
            column_types[c.name] = c.type
        return column_types
856042
def all_dataset_ids(self, reader_name=None, composites=False):
        
        try:
            if reader_name:
                readers = [self.readers[reader_name]]
            else:
                readers = self.readers.values()
        except (AttributeError, KeyError):
            raise KeyError("No reader '%s' found in scene" % reader_name)

        all_datasets = [dataset_id
                        for reader in readers
                        for dataset_id in reader.all_dataset_ids]
        if composites:
            all_datasets += self.all_composite_ids()
        return all_datasets
1296543
@SuppressWarnings("unchecked")
	public Map<String, String> getDatatypes() {
		if (datatypes == null) {
			datatypes = new DualHashBidiMap();
		}
		return datatypes;
	}
802646
def _maybe_dt_data(data, feature_names, feature_types):
    
    if not isinstance(data, DataTable):
        return data, feature_names, feature_types

    data_types_names = tuple(lt.name for lt in data.ltypes)
    bad_fields = [data.names[i]
                  for i, type_name in enumerate(data_types_names)
                  if type_name not in DT_TYPE_MAPPER]
    if bad_fields:
        msg = """DataFrame.types for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        feature_names = data.names

        # always return stypes for dt ingestion
        if feature_types is not None:
            raise ValueError('DataTable has own feature types, cannot pass them in')
        feature_types = np.vectorize(DT_TYPE_MAPPER2.get)(data_types_names)

    return data, feature_names, feature_types
1210971
def listPrimaryDatasets(self, primary_ds_name="", primary_ds_type=""):
        
        conn = self.dbi.connection()
        try:
            result = self.primdslist.execute(conn, primary_ds_name, primary_ds_type)
            if conn: conn.close()
            return result
        finally:
            if conn:
                conn.close()
625428
def check_dataset(dataset, mode):
  """"""

  names = [x['name'] for x in dataset.schema]
  types = [x['type'] for x in dataset.schema]
  if mode == 'train':
    if (set(['image_url', 'label']) != set(names) or any(t != 'STRING' for t in types)):
      raise ValueError('Invalid dataset. Expect only "image_url,label" STRING columns.')
  else:
    if (set(['image_url']) != set(names) and set(['image_url', 'label']) != set(names)) or \
            any(t != 'STRING' for t in types):
      raise ValueError('Invalid dataset. Expect only "image_url" or "image_url,label" ' +
                       'STRING columns.')
165078
def dataSetElementType(h5Dataset):
    
    dtype =  h5Dataset.dtype

    if dtype.names:
        return '<structured>'
    else:
        if dtype.metadata and 'vlen' in dtype.metadata:
            vlen_type = dtype.metadata['vlen']
            try:
                return "<vlen {}>".format(vlen_type.__name__)  # when vlen_type is a type
            except AttributeError: #
                return "<vlen {}>".format(vlen_type.name)      # when vlen_type is a dtype

    return str(dtype)
818182
def search_datasets(dataset_id=None,
                dataset_name=None,
                collection_name=None,
                data_type=None,
                unit_id=None,
                scenario_id=None,
                metadata_key=None,
                metadata_val=None,
                attr_id = None,
                type_id = None,
                unconnected = None,
                inc_metadata='N',
                inc_val = 'N',
                page_start = 0,
                page_size   = 2000,
                **kwargs):
    


    log.info("Searching datasets: \ndatset_id: %s,\n"
                                  "datset_name: %s,\n"
                                  "collection_name: %s,\n"
                                  "data_type: %s,\n"
                                  "unit_id: %s,\n"
                                  "scenario_id: %s,\n"
                                  "metadata_key: %s,\n"
                                  "metadata_val: %s,\n"
                                  "attr_id: %s,\n"
                                  "type_id: %s,\n"
                                  "unconnected: %s,\n"
                                  "inc_metadata: %s,\n"
                                  "inc_val: %s,\n"
                                  "page_start: %s,\n"
                                  "page_size: %s" % (dataset_id,
                dataset_name,
                collection_name,
                data_type,
                unit_id,
                scenario_id,
                metadata_key,
                metadata_val,
                attr_id,
                type_id,
                unconnected,
                inc_metadata,
                inc_val,
                page_start,
                page_size))

    if page_size is None:
        page_size = config.get('SEARCH', 'page_size', 2000)

    user_id = int(kwargs.get('user_id'))

    dataset_qry = db.DBSession.query(Dataset.id,
            Dataset.type,
            Dataset.unit_id,
            Dataset.name,
            Dataset.hidden,
            Dataset.cr_date,
            Dataset.created_by,
            DatasetOwner.user_id,
            null().label('metadata'),
            Dataset.value
    )

    #Dataset ID is unique, so there's no point using the other filters.
    #Only use other filters if the datset ID is not specified.
    if dataset_id is not None:
        dataset_qry = dataset_qry.filter(
            Dataset.id==dataset_id)

    else:
        if dataset_name is not None:
            dataset_qry = dataset_qry.filter(
                func.lower(Dataset.name).like("%%%s%%"%dataset_name.lower())
            )
        if collection_name is not None:
            dc = aliased(DatasetCollection)
            dci = aliased(DatasetCollectionItem)
            dataset_qry = dataset_qry.join(dc,
                        func.lower(dc.name).like("%%%s%%"%collection_name.lower())
                        ).join(dci,and_(
                            dci.collection_id == dc.id,
                            dci.dataset_id == Dataset.id))

        if data_type is not None:
            dataset_qry = dataset_qry.filter(
                func.lower(Dataset.type) == data_type.lower())

        #null is a valid unit, so we need a way for the searcher
        #to specify that they want to search for datasets with a null unit
        #rather than ignoring the unit. We use 'null' to do this.

        if unit_id is not None:
            dataset_qry = dataset_qry.filter(
                Dataset.unit_id == unit_id)

        if scenario_id is not None:
            dataset_qry = dataset_qry.join(ResourceScenario,
                                and_(ResourceScenario.dataset_id == Dataset.id,
                                ResourceScenario.scenario_id == scenario_id))

        if attr_id is not None:
            dataset_qry = dataset_qry.join(
                ResourceScenario, ResourceScenario.dataset_id == Dataset.id).join(
                ResourceAttr, and_(ResourceAttr.id==ResourceScenario.resource_attr_id,
                                  ResourceAttr.attr_id==attr_id))

        if type_id is not None:
            dataset_qry = dataset_qry.join(
                ResourceScenario, ResourceScenario.dataset_id == Dataset.id).join(
                ResourceAttr, ResourceAttr.id==ResourceScenario.resource_attr_id).join(
                TypeAttr, and_(TypeAttr.attr_id==ResourceAttr.attr_id, TypeAttr.type_id==type_id))

        if unconnected == 'Y':
            stmt = db.DBSession.query(distinct(ResourceScenario.dataset_id).label('dataset_id'),
                                literal_column("0").label('col')).subquery()
            dataset_qry = dataset_qry.outerjoin(
                stmt, stmt.c.dataset_id == Dataset.id)
            dataset_qry = dataset_qry.filter(stmt.c.col == None)
        elif unconnected == 'N':
            #The dataset has to be connected to something
            stmt = db.DBSession.query(distinct(ResourceScenario.dataset_id).label('dataset_id'),
                                literal_column("0").label('col')).subquery()
            dataset_qry = dataset_qry.join(
                stmt, stmt.c.dataset_id == Dataset.id)
        if metadata_key is not None and metadata_val is not None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.key).like("%%%s%%"%metadata_key.lower()),
                                func.lower(Metadata.value).like("%%%s%%"%metadata_val.lower())))
        elif metadata_key is not None and metadata_val is None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.key).like("%%%s%%"%metadata_key.lower())))
        elif metadata_key is None and metadata_val is not None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.value).like("%%%s%%"%metadata_val.lower())))

    #All datasets must be joined on dataset owner so only datasets that the
    #user can see are retrieved.
    dataset_qry = dataset_qry.outerjoin(DatasetOwner,
                                and_(DatasetOwner.dataset_id==Dataset.id,
                                DatasetOwner.user_id==user_id))

    dataset_qry = dataset_qry.filter(or_(Dataset.hidden=='N', and_(DatasetOwner.user_id is not None, Dataset.hidden=='Y')))

    log.info(str(dataset_qry))

    datasets = dataset_qry.all()

    log.info("Retrieved %s datasets", len(datasets))

    #page the datasets:
    if page_start + page_size > len(datasets):
        page_end = None
    else:
        page_end = page_start + page_size

    datasets = datasets[page_start:page_end]

    log.info("Datasets paged from result %s to %s", page_start, page_end)

    datasets_to_return = []
    for dataset_row in datasets:

        dataset_dict = dataset_row._asdict()



        if inc_val == 'N':
            dataset_dict['value'] = None
        else:
            #convert the value row into a string as it is returned as a binary
            if dataset_row.value is not None:
                dataset_dict['value'] = str(dataset_row.value)

        if inc_metadata=='Y':
            metadata = db.DBSession.query(Metadata).filter(Metadata.dataset_id==dataset_row.dataset_id).all()
            dataset_dict['metadata'] = metadata
        else:
            dataset_dict['metadata'] = []

        dataset = namedtuple('Dataset', dataset_dict.keys())(**dataset_dict)

        datasets_to_return.append(dataset)

    return datasets_to_return
309416
protected function getDataTypes(): array
    {
        $dataTypes = array_keys($this->module->dataTypes);

        // If include is specified.
        if (null !== $this->include) {
            $dataTypes = $this->applyIncludes($dataTypes);
        }

        // If there are exclusions.
        if (null !== $this->exclude) {
            $dataTypes = $this->applyExcludes($dataTypes);
        }

        //Import fields and usergroups again after all sources have been imported
        if (array_search('fields', $dataTypes) && count($dataTypes) > 1) {
            $dataTypes[] = 'fields';
            $dataTypes[] = 'userGroups';
        }

        return $dataTypes;
    }