Load dataset, use 20newsgroups dataset
199525
def load_data():
    ''''''
    digits = load_digits()
    X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, random_state=99, test_size=0.25)

    ss = StandardScaler()
    X_train = ss.fit_transform(X_train)
    X_test = ss.transform(X_test)

    return X_train, X_test, y_train, y_test
888176
def load_rocstories_dataset(dataset_path):
    
    with open(dataset_path, encoding='utf_8') as f:
        f = csv.reader(f)
        output = []
        next(f) # skip the first line
        for line in tqdm(f):
            output.append((' '.join(line[1:5]), line[5], line[6], int(line[-1])-1))
    return output
979047
def create_dataset(
            self, group, chunk_size, compression=None, compression_opts=None):
        """"""
        group.attrs['format'] = self.dformat
        super(Features, self)._create_dataset(
            group, chunk_size, compression, compression_opts)

        # TODO attribute declared outside __init__ is not safe. Used
        # because Labels.create_dataset need it.
        if chunk_size != 'auto':
            self.nb_per_chunk = nb_per_chunk(
                self.dtype.itemsize, self.dim, chunk_size)
        else:
            self.nb_per_chunk = 'auto'
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
98948
def _fetch_data(self, dataset, query={}, include_inactive_stations=False):
        
        data = []
        parameter = dataset
        station_dim = dataset.dimensions["station"]
        all_stations = station_dim.allowed_values
        # Step 1: Prepare query
        if "station" not in query:
            if include_inactive_stations:
                # Get all stations
                query["station"] = list(all_stations)
            else:
                # Get only active stations
                query["station"] = list(station_dim.active_stations())
        else:
            if not isinstance(query["station"], list):
                query["station"] = [query["station"]]
            # Make sure that the queried stations actually exist
            query["station"] = [ all_stations.get_by_label(x) for x in query["station"]]

        if "period" not in query:
            # TODO: I'd prepare to do dataset.get("period").allowed_values here
            query["period"] = PERIODS

        elif not isinstance(query["period"], list):
            query["period"] = [query["period"]]

        for period in query["period"]:
            if period not in PERIODS:
                msg = u"{} is not an allowed period".format(period)
                raise Exception(msg)


        # Step 3: Get data
        n_queries = len(query["station"]) * len(query["period"])
        counter = 0
        print("Fetching data with {} queries.".format(n_queries))
        for station in query["station"]:
            for period in query["period"]:
                url = dataset.url\
                    .replace(".json", "/station/{}/period/{}/data.csv"\
                        .format(station.key, period))
                print("/GET {} ".format(url))
                r = requests.get(url)

                if r.status_code == 200:
                    raw_data = DataCsv().from_string(r.content).to_dictlist()

                    # TODO: This is a very hard coded parse function
                    # Expects fixed start row and number of cols
                    for row in raw_data:
                        #timepoint = datetime.strptime(timepoint_str, "%Y-%m-%d %H:%M:%S")
                        value_col = parameter.id.split(",")[0]
                        value = float(row[value_col])

                        row["parameter"] = parameter.id
                        row["station"] = station.label
                        row["station_key"] = station.key
                        row["period"] = period

                        row.pop(value_col,None)

                        datapoint = Result(value, row)

                        yield datapoint

                elif r.status_code == 404:
                    print("Warning no data at {}".format(url))
                else:
                    raise Exception("Connection error for {}".format(url))
1308176
def load_remote_dataset(self, ds_str):
        '''
        
        '''

        if opendap.is_opendap(ds_str):
            return Dataset(ds_str)
        else:
            # Check if the HTTP response is XML, if it is, it's likely SOS so
            # we'll attempt to parse the response as SOS
            response = requests.get(ds_str, allow_redirects=True)
            if 'text/xml' in response.headers['content-type']:
                return self.process_doc(response.content)

            raise ValueError("Unknown service with content-type: {}".format(response.headers['content-type']))
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
1154967
def load(self, dataset_keys, previous_datasets=None):
        """"""
        all_datasets = previous_datasets or DatasetDict()
        datasets = DatasetDict()

        # Include coordinates in the list of datasets to load
        dsids = [self.get_dataset_key(ds_key) for ds_key in dataset_keys]
        coordinates = self._get_coordinates_for_dataset_keys(dsids)
        all_dsids = list(set().union(*coordinates.values())) + dsids

        for dsid in all_dsids:
            if dsid in all_datasets:
                continue
            coords = [all_datasets.get(cid, None)
                      for cid in coordinates.get(dsid, [])]
            ds = self._load_dataset_with_area(dsid, coords)
            if ds is not None:
                all_datasets[dsid] = ds
                if dsid in dsids:
                    datasets[dsid] = ds
        self._load_ancillary_variables(all_datasets)

        return datasets
402497
def download_dataset(tournament, unzip):
    """"""
    click.echo(napi.download_current_dataset(tournament=tournament, unzip=unzip))
663476
def download_data(dataset_name=None, prompt=prompt_stdin):
    """"""
        
    dr = data_resources[dataset_name]
    if not authorize_download(dataset_name, prompt=prompt):
        raise Exception("Permission to download data set denied.")
    
    if 'suffices' in dr:
        for url, files, suffices in zip(dr['urls'], dr['files'], dr['suffices']):
            for file, suffix in zip(files, suffices):
                download_url(url=os.path.join(url,file),
                             dir_name = data_path,
                             store_directory=dataset_name,
                             suffix=suffix)
    elif 'dirs' in dr:
        for url, dirs, files in zip(dr['urls'], dr['dirs'], dr['files']):
            for file, dir in zip(files, dirs):
                print(file, dir)
                download_url(
                    url=os.path.join(url,dir,file),
                    dir_name = data_path,
                    store_directory=os.path.join(dataset_name,dir)
                    )
    else:
        for url, files in zip(dr['urls'], dr['files']):
            for file in files:
                download_url(
                    url=os.path.join(url,file),
                    dir_name = data_path,
                    store_directory=dataset_name
                    )
    return True
209122
function(dataset){
    this.sources.push(dataset);
    // add event listeners to source
    if (this.listen.source)
      dataset.addHandler(this.listen.source, this);

    // process new dataset objects and update member map
    var memberMap = this.members_;
    for (var objectId in dataset.items_)
    {
      // check: is this object already known
      if (memberMap[objectId])
      {
        // item exists -> increase dataset links count
        memberMap[objectId].count++;
      }
      else
      {
        // add to source map
        memberMap[objectId] = {
          count: 1,
          isMember: false,
          object: dataset.items_[objectId]
        };
      }
    }

    return true;
  }
293862
def create_table(group, name, dtype, **attributes):
    """"""
    dset = group.create_dataset(
        name, shape=(0,), dtype=dtype, maxshape=(None,))
    set_attributes(dset, **attributes)
    return dset
343264
def dataset(self):
        """"""
        data = tablib.Dataset()
        data.headers = self.keys()

        row = _reduce_datetimes(self.values())
        data.append(row)

        return data
407303
def _load_data(batch, targets, major_axis):
    """"""
    if isinstance(batch, list):
        new_batch = []
        for i in range(len(targets)):
            new_batch.append([b.data[i] for b in batch])
        new_targets = [[dst for _, dst in d_target] for d_target in targets]
        _load_general(new_batch, new_targets, major_axis)
    else:
        _load_general(batch.data, targets, major_axis)
420198
def get_data(
    dataset,
    query=None,
    crs="epsg:4326",
    bounds=None,
    sortby=None,
    pagesize=10000,
    max_workers=5,
):
    
    param_dicts = define_request(dataset, query, crs, bounds, sortby, pagesize)

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        results = executor.map(make_request, param_dicts)

    outjson = dict(type="FeatureCollection", features=[])
    for result in results:
        outjson["features"] += result
    return outjson
323627
def insertDatasetWOannex(self, dataset, blockcontent, otptIdList, conn,
                             insertDataset = True, migration = False):
        

        tran = conn.begin()
        try:
            #8 Finally, we have everything to insert a dataset
            if insertDataset:
                # Then we have to get a new dataset ID
                dataset['dataset_id'] = self.datasetid.execute(conn,
                                                dataset['dataset'])
                if dataset['dataset_id'] <= 0:
                    dataset['dataset_id'] = self.sm.increment(conn, "SEQ_DS")
                    if not migration:
                        dataset['last_modified_by'] = dbsUtils().getCreateBy()
                        dataset['create_by'] = dbsUtils().getCreateBy()
                        dataset['creation_date'] = dataset.get('creation_date', dbsUtils().getTime())
                        dataset['last_modification_date'] = dataset.get('last_modification_date', dbsUtils().getTime())
                    dataset['xtcrosssection'] = dataset.get('xtcrosssection', None)
                    dataset['prep_id'] = dataset.get('prep_id', None)
                    try:
                        self.datasetin.execute(conn, dataset, tran)
                    except exceptions.IntegrityError as ei:
                        if str(ei).find("ORA-00001") != -1 or str(ei).lower().find("duplicate") !=-1:
                            if conn.closed:
                                conn = self.dbi.connection()
                            dataset['dataset_id'] = self.datasetid.execute(conn, dataset['dataset'])
                            if dataset['dataset_id'] <= 0:
                                if tran:tran.rollback()
                                if conn:conn.close()
                                dbsExceptionHandler('dbsException-conflict-data',
                                                    'Dataset/[processed DS]/[dataset access type] not yet inserted by concurrent insert. ',
                                                    self.logger.exception,
                                                    'Dataset/[processed DS]/[dataset access type] not yet inserted by concurrent insert. '+ str(ei))
                        elif str(ei).find("ORA-01400") > -1:
                            if tran:tran.rollback()
                            if conn:conn.close()
                            dbsExceptionHandler('dbsException-missing-data',
                                'Missing data when insert Datasets. ',
                                self.logger.exception,
                                'Missing data when insert Datasets. '+ str(ei))
                        else:
                            if tran: tran.rollback()
                            if conn: conn.close()
                            dbsExceptionHandler('dbsException-invalid-input2',
                            'Invalid data when insert Datasets. ',
                            self.logger.exception,
                            'Invalid data when insert Datasets. '+ str(ei))

                    except Exception:
                        #should catch all above exception to rollback. YG Jan 17, 2013
                        if tran:tran.rollback()
                        if conn:conn.close()
                        raise

            #9 Fill Dataset Parentage
            #All parentage are deduced from file parentage.

            #10 Before we commit, make dataset and output module configuration
            #mapping.  We have to try to fill the map even if dataset is
            #already in dest db
            for c in otptIdList:
                try:
                    dcObj = {
                             'dataset_id' : dataset['dataset_id'],
                             'output_mod_config_id' : c }
                    self.dcin.execute(conn, dcObj, tran)
                except exceptions.IntegrityError as ei:
                    #FIXME YG 01/17/2013
                    if (str(ei).find("ORA-00001") != -1 and str(ei).find("TUC_DC_1") != -1) or \
                            str(ei).lower().find("duplicate")!=-1:
                    #ok, already in db
                    #FIXME: What happens when there are partially in db?
                    #YG 11/17/2010
                        pass
                    else:
                        if tran:tran.rollback()
                        if conn:conn.close()
                        dbsExceptionHandler('dbsException-invalid-input2',
                            'Invalid data when insert dataset_configs. ',
                            self.logger.exception,
                            'Invalid data when insert dataset_configs. '+ str(ei))
                except Exception as ex:
                    if tran:tran.rollback()
                    if conn:conn.close()
                    raise
            #Now commit everything.
            tran.commit()
        except exceptions.IntegrityError as ei:
            # Then is it already in the database?
            # Not really. We have to check it again. YG Jan 17, 2013
            # we don't check the unique key here, since there are more than one unique key might
            # be violated: such as data_tier, processed_dataset, dataset_access_types.
            if str(ei).find("ORA-00001") != -1 or str(ei).lower().find("duplicate")!=-1:
                # For now, we assume most cases are the same dataset was instered by different thread. If not,
                # one has to call the insert dataset again. But we think this is a rare case and let the second
                # DBSBlockInsert call fix it if it happens.
                if conn.closed:
                    conn = self.dbi.connection()
                dataset_id = self.datasetid.execute(conn, dataset['dataset'])
                if dataset_id <= 0:
                    dbsExceptionHandler('dbsException-conflict-data',
                                        'Dataset not yet inserted by concurrent insert',
                                        self.logger.exception,
                                        'Dataset not yet inserted by concurrent insert')

                else:
                    dataset['dataset_id'] = dataset_id
            else:
                if tran:tran.rollback()
                if conn:conn.close()
                dbsExceptionHandler('dbsException-invalid-input2',
                    'Invalid data when insert Datasets. ',
                    self.logger.exception,
                    'Invalid data when insert Datasets. '+ str(ei))
        except Exception as ex:
            if tran:tran.rollback()
            if conn:conn.close()
            raise
        finally:
            if tran:tran.rollback()
            if conn:conn.close()
        return dataset['dataset_id']
321706
def addDataset(self):
        
        self._openRepo()
        dataset = datasets.Dataset(self._args.datasetName)
        dataset.setDescription(self._args.description)
        dataset.setAttributes(json.loads(self._args.attributes))
        self._updateRepo(self._repo.insertDataset, dataset)