Lists all primary datasets if pattern is not provided.
181662
def execute(self, conn, run_num=-1, logical_file_name="", block_name="", dataset="", trans=False):
        
        sql = self.sql
        binds = {}
	if logical_file_name and "%" not in logical_file_name:
	    sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
		    WHERE FILES.LOGICAL_FILE_NAME = :logical_file_name"""%(self.owner)
	    binds["logical_file_name"] = logical_file_name
	elif block_name and "%" not in block_name:
            sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
		    inner join %sBLOCKS BLOCKS on BLOCKS.BLOCK_ID = FILES.BLOCK_ID
		    WHERE BLOCKS.BLOCK_NAME = :block_name """%(self.owner, self.owner)
            binds["block_name"] =  block_name
	elif dataset and "%" not in dataset:
	    sql += """ inner join %sFILES FILES on FILES.FILE_ID = FL.FILE_ID
	    inner join %sDATASETS DATASETS on DATASETS.DATASET_ID = FILES.DATASET_ID
	    WHERE DATASETS.DATASET = :dataset """%(self.owner, self.owner)
	    binds["dataset"] = dataset
	else:
	    pass
        
	if run_num != -1:
            andorwhere = ("WHERE", "AND")["WHERE" in sql]
            run_list = []
            wheresql_run_list = ''
            wheresql_run_range = ''
            #
            for r in parseRunRange(run_num):
                if isinstance(r, basestring) or isinstance(r, int) or isinstance(r, long):
                    run_list.append(str(r))
                if isinstance(r, run_tuple):
                    if r[0] == r[1]:
                        dbsExceptionHandler('dbsException-invalid-input', "DBS run_num range must be apart at least by 1.", self.logger.exception)
                    wheresql_run_range = " FL.RUN_NUM between :minrun and :maxrun "
                    binds.update({"minrun":r[0]})
                    binds.update({"maxrun":r[1]})
            # 
            if run_list:
                wheresql_run_list = " fl.RUN_NUM in (SELECT TOKEN FROM TOKEN_GENERATOR) "
                run_generator, run_binds = create_token_generator(run_list)
                sql =  "{run_generator}".format(run_generator=run_generator) + sql
                binds.update(run_binds)

            if wheresql_run_range and wheresql_run_list:
                sql += " %s (" %andorwhere    + wheresql_run_range + " or " +  wheresql_run_list + " )"
            elif wheresql_run_range and not wheresql_run_list:
                sql += " %s " %andorwhere  + wheresql_run_range
            elif not wheresql_run_range and wheresql_run_list:
                sql += " %s " %andorwhere  + wheresql_run_list
        #self.logger.debug(sql)
	cursors = self.dbi.processData(sql, binds, conn, transaction=trans, returnCursor=True)
        result=[]
        for c in cursors:
            result.extend(self.formatCursor(c, size=100))
        return result
301233
def getDatasetByName(self, name):
        
        if name not in self._datasetNameMap:
            raise exceptions.DatasetNameNotFoundException(name)
        return self._datasetNameMap[name]
1154967
def load(self, dataset_keys, previous_datasets=None):
        """"""
        all_datasets = previous_datasets or DatasetDict()
        datasets = DatasetDict()

        # Include coordinates in the list of datasets to load
        dsids = [self.get_dataset_key(ds_key) for ds_key in dataset_keys]
        coordinates = self._get_coordinates_for_dataset_keys(dsids)
        all_dsids = list(set().union(*coordinates.values())) + dsids

        for dsid in all_dsids:
            if dsid in all_datasets:
                continue
            coords = [all_datasets.get(cid, None)
                      for cid in coordinates.get(dsid, [])]
            ds = self._load_dataset_with_area(dsid, coords)
            if ds is not None:
                all_datasets[dsid] = ds
                if dsid in dsids:
                    datasets[dsid] = ds
        self._load_ancillary_variables(all_datasets)

        return datasets
187655
def trace_filter (patterns):
    """"""
    if patterns is None:
        _trace_filter.clear()
    else:
        _trace_filter.update(re.compile(pat) for pat in patterns)
186225
def getDatasetByName(self, name):
        
        
        result = None
        for dataset in self.__datasets:
            datasetName = dataset.getName()
            if datasetName == name:
                result = dataset
                break
        return result
1133325
public java.util.List<Dataset> getDatasets() {
        if (datasets == null) {
            datasets = new com.amazonaws.internal.SdkInternalList<Dataset>();
        }
        return datasets;
    }
830973
def get_by_index(self, index):
        
        if index >= len(self._datasets):
            raise DataInvalidIndex('A dataset with index {} does not exist'.format(index))

        return self._datasets[index]
1086307
def _filter_by_pattern(self, pattern):
        """"""
        try:
            _len = len(pattern)
        except TypeError:
            raise TypeError("pattern is not a list of Booleans. Got {}".format(
                type(pattern)))
        _filt_values = [d for i, d in enumerate(self._values) if pattern[i % _len]]
        _filt_datetimes = [d for i, d in enumerate(self.datetimes) if pattern[i % _len]]
        return _filt_values, _filt_datetimes
125476
def listFiles(self, dataset="", block_name="", logical_file_name="",
                  release_version="", pset_hash="", app_name="",
                  output_module_label="",  run_num=-1,
                  origin_site_name="", lumi_list=[], detail=False, validFileOnly=0, sumOverLumi=0, input_body=-1):
        
        if input_body != -1 :
	    try:
                logical_file_name = input_body.get("logical_file_name", "")
                run_num = input_body.get("run_num", -1)
                validFileOnly = input_body.get("validFileOnly", 0)
                sumOverLumi = input_body.get("sumOverLumi", 0)  
		detail = input_body.get("detail", False)
                block_name = input_body.get("block_name", "")
		dataset = input_body.get("dataset", "") 
		release_version = input_body.get("release_version", "")
		pset_hash = input_body.get("pset_hash", "")
	        app_name = input_body.get("app_name", "")
		output_module_label = input_body.get("output_module_label", "")
		origin_site_name = input_body.get("origin_site_name", "")
		lumi_list = input_body.get("lumi_list", [])	
            except cjson.DecodeError as de:
                msg = "business/listFilss POST call requires at least dataset, block_name, or a list of logical_file_name %s" % de
                dbsExceptionHandler('dbsException-invalid-input', "Invalid input", self.logger.exception, msg)

        if ('%' in block_name):
            dbsExceptionHandler('dbsException-invalid-input', "You must specify exact block name not a pattern", self.logger.exception)
        elif ('%' in dataset):
	    print("***** in dataset name")
            dbsExceptionHandler('dbsException-invalid-input', " You must specify exact dataset name not a pattern", self.logger.exception)
        elif (not dataset  and not block_name and (not logical_file_name or '%'in logical_file_name) ):
            dbsExceptionHandler('dbsException-invalid-input', """You must specify one of the parameter groups:  \
                    non-pattern dataset, \
                    non-pattern block , non-pattern dataset with lfn ,\
                    non-pattern block with lfn or no-pattern lfn, \
		    non-patterned lfn list .""", self.logger.exception)
        elif (lumi_list and len(lumi_list) != 0):
            if run_num==-1:
                dbsExceptionHandler('dbsException-invalid-input', "Lumi list must accompany A single run number, \
                        use run_num=123", self.logger.exception)
            elif isinstance(run_num, basestring):
                try:
                    run_num = int(run_num)
                except:
                    dbsExceptionHandler('dbsException-invalid-input', "Lumi list must accompany A single run number,\
                        use run_num=123", self.logger.exception)
            elif isinstance(run_num, list):
                if len(run_num) == 1:
                    try:
                        run_num = int(run_num[0])
                    except:
                        dbsExceptionHandler('dbsException-invalid-input', "Lumi list must accompany A single run number,\
                            use run_num=123", self.logger.exception)
                else:
                    dbsExceptionHandler('dbsException-invalid-input', "Lumi list must accompany A single run number,\
                        use run_num=123", self.logger.exception)
	else:
            pass
        with self.dbi.connection() as conn:
            dao = (self.filebrieflist, self.filelist)[detail]
            for item in dao.execute(conn, dataset, block_name, logical_file_name, release_version, pset_hash, app_name,
                            output_module_label, run_num, origin_site_name, lumi_list, validFileOnly, sumOverLumi):

                yield item      # we need to yield while connection is open
107923
def get_collections(self, data):
        """"""
        collections = self._filter_queryset('view_collection', data.collection_set.all())

        from .collection import CollectionSerializer

        class CollectionWithoutDataSerializer(WithoutDataSerializerMixin, CollectionSerializer):
            """Collection without data field serializer."""

        return self._serialize_items(CollectionWithoutDataSerializer, 'collections', collections)
1052033
def is_dataset(ds):
  """"""
  import tensorflow as tf
  from tensorflow_datasets.core.utils import py_utils
  dataset_types = [tf.data.Dataset]
  v1_ds = py_utils.rgetattr(tf, "compat.v1.data.Dataset", None)
  v2_ds = py_utils.rgetattr(tf, "compat.v2.data.Dataset", None)
  if v1_ds is not None:
    dataset_types.append(v1_ds)
  if v2_ds is not None:
    dataset_types.append(v2_ds)
  return isinstance(ds, tuple(dataset_types))
1132938
def listDatabases(self):
        """"""
        iter = self._jcatalog.listDatabases().toLocalIterator()
        databases = []
        while iter.hasNext():
            jdb = iter.next()
            databases.append(Database(
                name=jdb.name(),
                description=jdb.description(),
                locationUri=jdb.locationUri()))
        return databases
818182
def search_datasets(dataset_id=None,
                dataset_name=None,
                collection_name=None,
                data_type=None,
                unit_id=None,
                scenario_id=None,
                metadata_key=None,
                metadata_val=None,
                attr_id = None,
                type_id = None,
                unconnected = None,
                inc_metadata='N',
                inc_val = 'N',
                page_start = 0,
                page_size   = 2000,
                **kwargs):
    


    log.info("Searching datasets: \ndatset_id: %s,\n"
                                  "datset_name: %s,\n"
                                  "collection_name: %s,\n"
                                  "data_type: %s,\n"
                                  "unit_id: %s,\n"
                                  "scenario_id: %s,\n"
                                  "metadata_key: %s,\n"
                                  "metadata_val: %s,\n"
                                  "attr_id: %s,\n"
                                  "type_id: %s,\n"
                                  "unconnected: %s,\n"
                                  "inc_metadata: %s,\n"
                                  "inc_val: %s,\n"
                                  "page_start: %s,\n"
                                  "page_size: %s" % (dataset_id,
                dataset_name,
                collection_name,
                data_type,
                unit_id,
                scenario_id,
                metadata_key,
                metadata_val,
                attr_id,
                type_id,
                unconnected,
                inc_metadata,
                inc_val,
                page_start,
                page_size))

    if page_size is None:
        page_size = config.get('SEARCH', 'page_size', 2000)

    user_id = int(kwargs.get('user_id'))

    dataset_qry = db.DBSession.query(Dataset.id,
            Dataset.type,
            Dataset.unit_id,
            Dataset.name,
            Dataset.hidden,
            Dataset.cr_date,
            Dataset.created_by,
            DatasetOwner.user_id,
            null().label('metadata'),
            Dataset.value
    )

    #Dataset ID is unique, so there's no point using the other filters.
    #Only use other filters if the datset ID is not specified.
    if dataset_id is not None:
        dataset_qry = dataset_qry.filter(
            Dataset.id==dataset_id)

    else:
        if dataset_name is not None:
            dataset_qry = dataset_qry.filter(
                func.lower(Dataset.name).like("%%%s%%"%dataset_name.lower())
            )
        if collection_name is not None:
            dc = aliased(DatasetCollection)
            dci = aliased(DatasetCollectionItem)
            dataset_qry = dataset_qry.join(dc,
                        func.lower(dc.name).like("%%%s%%"%collection_name.lower())
                        ).join(dci,and_(
                            dci.collection_id == dc.id,
                            dci.dataset_id == Dataset.id))

        if data_type is not None:
            dataset_qry = dataset_qry.filter(
                func.lower(Dataset.type) == data_type.lower())

        #null is a valid unit, so we need a way for the searcher
        #to specify that they want to search for datasets with a null unit
        #rather than ignoring the unit. We use 'null' to do this.

        if unit_id is not None:
            dataset_qry = dataset_qry.filter(
                Dataset.unit_id == unit_id)

        if scenario_id is not None:
            dataset_qry = dataset_qry.join(ResourceScenario,
                                and_(ResourceScenario.dataset_id == Dataset.id,
                                ResourceScenario.scenario_id == scenario_id))

        if attr_id is not None:
            dataset_qry = dataset_qry.join(
                ResourceScenario, ResourceScenario.dataset_id == Dataset.id).join(
                ResourceAttr, and_(ResourceAttr.id==ResourceScenario.resource_attr_id,
                                  ResourceAttr.attr_id==attr_id))

        if type_id is not None:
            dataset_qry = dataset_qry.join(
                ResourceScenario, ResourceScenario.dataset_id == Dataset.id).join(
                ResourceAttr, ResourceAttr.id==ResourceScenario.resource_attr_id).join(
                TypeAttr, and_(TypeAttr.attr_id==ResourceAttr.attr_id, TypeAttr.type_id==type_id))

        if unconnected == 'Y':
            stmt = db.DBSession.query(distinct(ResourceScenario.dataset_id).label('dataset_id'),
                                literal_column("0").label('col')).subquery()
            dataset_qry = dataset_qry.outerjoin(
                stmt, stmt.c.dataset_id == Dataset.id)
            dataset_qry = dataset_qry.filter(stmt.c.col == None)
        elif unconnected == 'N':
            #The dataset has to be connected to something
            stmt = db.DBSession.query(distinct(ResourceScenario.dataset_id).label('dataset_id'),
                                literal_column("0").label('col')).subquery()
            dataset_qry = dataset_qry.join(
                stmt, stmt.c.dataset_id == Dataset.id)
        if metadata_key is not None and metadata_val is not None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.key).like("%%%s%%"%metadata_key.lower()),
                                func.lower(Metadata.value).like("%%%s%%"%metadata_val.lower())))
        elif metadata_key is not None and metadata_val is None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.key).like("%%%s%%"%metadata_key.lower())))
        elif metadata_key is None and metadata_val is not None:
            dataset_qry = dataset_qry.join(Metadata,
                                and_(Metadata.dataset_id == Dataset.id,
                                func.lower(Metadata.value).like("%%%s%%"%metadata_val.lower())))

    #All datasets must be joined on dataset owner so only datasets that the
    #user can see are retrieved.
    dataset_qry = dataset_qry.outerjoin(DatasetOwner,
                                and_(DatasetOwner.dataset_id==Dataset.id,
                                DatasetOwner.user_id==user_id))

    dataset_qry = dataset_qry.filter(or_(Dataset.hidden=='N', and_(DatasetOwner.user_id is not None, Dataset.hidden=='Y')))

    log.info(str(dataset_qry))

    datasets = dataset_qry.all()

    log.info("Retrieved %s datasets", len(datasets))

    #page the datasets:
    if page_start + page_size > len(datasets):
        page_end = None
    else:
        page_end = page_start + page_size

    datasets = datasets[page_start:page_end]

    log.info("Datasets paged from result %s to %s", page_start, page_end)

    datasets_to_return = []
    for dataset_row in datasets:

        dataset_dict = dataset_row._asdict()



        if inc_val == 'N':
            dataset_dict['value'] = None
        else:
            #convert the value row into a string as it is returned as a binary
            if dataset_row.value is not None:
                dataset_dict['value'] = str(dataset_row.value)

        if inc_metadata=='Y':
            metadata = db.DBSession.query(Metadata).filter(Metadata.dataset_id==dataset_row.dataset_id).all()
            dataset_dict['metadata'] = metadata
        else:
            dataset_dict['metadata'] = []

        dataset = namedtuple('Dataset', dataset_dict.keys())(**dataset_dict)

        datasets_to_return.append(dataset)

    return datasets_to_return
1292784
def find_all(self, cls):
        """"""
        final_results = []
        table = self.get_class_table(cls)
        for db_result in table.scan():
            obj = cls.from_data(db_result['value'])
            final_results.append(obj)

        return final_results
59896
def get_datasets(dataset_ids,**kwargs):
    

    user_id = int(kwargs.get('user_id'))
    datasets = []
    if len(dataset_ids) == 0:
        return []
    try:
        dataset_rs = db.DBSession.query(Dataset.id,
                Dataset.type,
                Dataset.unit_id,
                Dataset.name,
                Dataset.hidden,
                Dataset.cr_date,
                Dataset.created_by,
                DatasetOwner.user_id,
                null().label('metadata'),
                case([(and_(Dataset.hidden=='Y', DatasetOwner.user_id is not None), None)],
                        else_=Dataset.value).label('value')).filter(
                Dataset.id.in_(dataset_ids)).outerjoin(DatasetOwner,
                                    and_(DatasetOwner.dataset_id==Dataset.id,
                                    DatasetOwner.user_id==user_id)).all()

        #convert the value row into a string as it is returned as a binary
        for dataset_row in dataset_rs:
            dataset_dict = dataset_row._asdict()

            if dataset_row.value is not None:
                dataset_dict['value'] = str(dataset_row.value)

            if dataset_row.hidden == 'N' or (dataset_row.hidden == 'Y' and dataset_row.user_id is not None):
                metadata = db.DBSession.query(Metadata).filter(Metadata.dataset_id == dataset_row.id).all()
                dataset_dict['metadata'] = metadata
            else:
                dataset_dict['metadata'] = []

            datasets.append(namedtuple('Dataset', dataset_dict.keys())(**dataset_dict))


    except NoResultFound:
        raise ResourceNotFoundError("Datasets not found.")

    return datasets
9901
def search_datasets(
        self,
        license=None,
        format=None,
        query=None,
        featured=None,
        owner=None,
        organization=None,
        badge=None,
        reuses=None,
        page_size=20,
        x_fields=None,
    ):
        """"""
        # handling request parameters
        payload = {"badge": badge, "size": page_size, "X-Fields": x_fields}

        # search request
        # head = {"X-API-KEY": self.api_key}
        search_url = "{}/datasets".format(
            self.base_url,
            # org_id,
            # page_size
        )

        search_req = requests.get(
            search_url,
            # headers=head,
            params=payload,
        )

        # serializing result into dict and storing resources in variables
        logger.debug(search_req.url)
        return search_req.json()
867725
function list_patterns() {
	console.log("Available patterns:");
	console.log();

	for(let pattern of MHControl.patternNames) {
		console.log(pattern);
	}
}