Load or create dataset
499728
def load_data(train_path='./data/regression.train', test_path='./data/regression.test'):
    '''
    
    '''
    print('Load data...')
    df_train = pd.read_csv(train_path, header=None, sep='\t')
    df_test = pd.read_csv(test_path, header=None, sep='\t')
    num = len(df_train)
    split_num = int(0.9 * num)

    y_train = df_train[0].values
    y_test = df_test[0].values
    y_eval = y_train[split_num:]
    y_train = y_train[:split_num]

    X_train = df_train.drop(0, axis=1).values
    X_test = df_test.drop(0, axis=1).values
    X_eval = X_train[split_num:, :]
    X_train = X_train[:split_num, :]

    # create dataset for lightgbm
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_eval, y_eval, reference=lgb_train)

    return lgb_train, lgb_eval, X_test, y_test
162352
def create_preprocessed_dataset(path_to_data, outputpath, preprocessing_queue):
    """"""
    # Log everything
    logging.info("Data soure %s", path_to_data)
    logging.info("Output will be stored in %s", outputpath)
    tmp = "Preprocessing Queue:\n"
    for preprocessing_class in preprocessing_queue:
        tmp += str(preprocessing_class) + "\n"
    logging.info(tmp)
    # Load from pickled file
    if not os.path.isfile(path_to_data):
        logging.info(("'%s' does not exist. Please either abort this script "
                      "or update the data location."), path_to_data)
        raw_dataset_path = utils.choose_raw_dataset()
        # Get project-relative path
        raw_dataset_path = "raw-datasets" + \
                           raw_dataset_path.split("raw-datasets")[1]
        print(raw_dataset_path)
        sys.exit()  # TODO: Update model!
    logging.info("Start loading data...")
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    logging.info("Start applying preprocessing methods")
    start_time = time.time()
    for i, raw_dataset in enumerate(raw_datasets):
        if i % 10 == 0 and i > 0:
            utils.print_status(len(raw_datasets), i, start_time)
        # Do the work
        raw_dataset['handwriting'].preprocessing(preprocessing_queue)
    sys.stdout.write("\r%0.2f%% (done)\033[K\n" % (100))
    print("")
    pickle.dump({'handwriting_datasets': raw_datasets,
                 'formula_id2latex': loaded['formula_id2latex'],
                 'preprocessing_queue': preprocessing_queue},
                open(outputpath, "wb"),
                2)
1117685
def load_local_dataset(self, ds_str):
        '''
        
        '''
        if cdl.is_cdl(ds_str):
            ds_str = self.generate_dataset(ds_str)

        if netcdf.is_netcdf(ds_str):
            return MemoizedDataset(ds_str)

        # Assume this is just a Generic File if it exists
        if os.path.isfile(ds_str):
            return GenericFile(ds_str)

        raise ValueError("File is an unknown format")
427548
def CreateDataset(self):
    """"""
    body = {
        "datasetReference": {
            "datasetId": self.dataset_id,
            "description": "Data exported from GRR",
            "friendlyName": "GRRExportData",
            "projectId": self.project_id
        }
    }
    result = self.service.datasets().insert(
        projectId=self.project_id, body=body).execute()
    self.datasets[self.dataset_id] = result
    return result
227353
def open(dataset_dir, access_mode=READ_ONLY_ACCESS):
        
        # check access mode
        if access_mode == WRITE_ACCESS:
            raise ValueError('Cannot open a dataset with write-only access')

        # read config
        try:
            # json load
            config_filename = os.path.join(dataset_dir, 'config.json')
            config = json.load(open(config_filename, 'r'))
        except:
            # YAML load
            config_filename = os.path.join(dataset_dir, 'config.yaml')
            config = YamlConfig(config_filename)

        # open dataset
        dataset = TensorDataset(dataset_dir, config, access_mode=access_mode)
        return dataset
75074
def create_dataset(self, in_stream):
        
        assert XLS_IMPORT
        xls_book = xlrd.open_workbook(file_contents=in_stream)
        dataset = tablib.Dataset()
        sheet = xls_book.sheets()[0]

        dataset.headers = sheet.row_values(0)
        for i in range(1, sheet.nrows):
            dataset.append(sheet.row_values(i))
        return dataset
1003029
def create_dataset(self, in_stream):
        
        assert XLSX_IMPORT
        from io import BytesIO
        xlsx_book = openpyxl.load_workbook(BytesIO(in_stream), read_only=True)

        dataset = tablib.Dataset()
        sheet = xlsx_book.active

        # obtain generator
        rows = sheet.rows
        dataset.headers = [cell.value for cell in next(rows)]

        for row in rows:
            row_values = [cell.value for cell in row]
            dataset.append(row_values)
        return dataset
618105
def upload_dataset(self, dataset, dataset_name, schema_path=None):
        

        url = self.address + "/datasets/" + dataset_name + "/uploadSample"
        header = self.__check_authentication()

        fields = dict()
        remove = False
        if isinstance(dataset, GDataframe):
            tmp_path = TempFileManager.get_new_dataset_tmp_folder()
            dataset.to_dataset_files(local_path=tmp_path)
            dataset = tmp_path
            remove = True

        # a path is provided
        if not isinstance(dataset, str):
            raise TypeError("Dataset can be a path or a GDataframe. {} was passed".format(type(dataset)))

        file_paths, schema_path_found = Loader.get_file_paths(dataset)
        if schema_path is None:
            schema_path = schema_path_found
        fields['schema'] = (os.path.basename(schema_path), open(schema_path, "rb"), 'application/octet-stream')
        for i, file in enumerate(file_paths):
            fields["file"+str(i + 1)] = (os.path.basename(file), open(file, "rb"), 'application/octet-stream')

        encoder = MultipartEncoder(fields)
        callback = create_callback(encoder, len(fields))

        m_encoder = MultipartEncoderMonitor(encoder, callback)
        header['Content-Type'] = m_encoder.content_type

        self.logger.debug("Uploading dataset at {} with name {}".format(dataset, dataset_name))

        response = requests.post(url, data=m_encoder,
                                 headers=header)

        # closing files
        for fn in fields.keys():
            _, f, _ = fields[fn]
            f.close()

        if response.status_code != 200:
            raise ValueError("Code {}: {}".format(response.status_code, response.content))

        if remove:
            TempFileManager.delete_tmp_dataset(dataset)
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
1167262
def dataset(self, ref, load_all=False, exception=True):
        """"""
        return self.database.dataset(ref, load_all=load_all, exception=exception)
1320440
def open_dataset(self, service):
        """"""

        if not self.dataset:
            path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
            self.dataset = netCDF4.Dataset(path, 'r')
        return self.dataset
820519
def __load(self, path):
        """"""
        try:
            path = os.path.abspath(path)
            with open(path, 'rb') as df:
                # loaded_dataset = pickle.load(df)
                self.__data, self.__classes, self.__labels, \
                self.__dtype, self.__description, \
                self.__num_features, self.__feature_names = pickle.load(df)

            # ensure the loaded dataset is valid
            self.__validate(self.__data, self.__classes, self.__labels)

        except IOError as ioe:
            raise IOError('Unable to read the dataset from file: {}', format(ioe))
        except:
            raise
606066
def main(folder):
    """"""
    raw_datapath, outputpath, p_queue = get_parameters(folder)
    create_preprocessed_dataset(raw_datapath, outputpath, p_queue)
    utils.create_run_logfile(folder)
203023
def create(self):
        """"""
        b0 = QGroupBox('Dataset')
        form = QFormLayout()
        b0.setLayout(form)

        open_rec = QPushButton('Open Dataset...')
        open_rec.clicked.connect(self.open_dataset)
        open_rec.setToolTip('Click here to open a new recording')
        self.idx_filename = open_rec
        self.idx_s_freq = QLabel('')
        self.idx_n_chan = QLabel('')
        self.idx_start_time = QLabel('')
        self.idx_end_time = QLabel('')

        form.addRow('Filename:', self.idx_filename)
        form.addRow('Sampl. Freq:', self.idx_s_freq)
        form.addRow('N. Channels:', self.idx_n_chan)
        form.addRow('Start Time: ', self.idx_start_time)
        form.addRow('End Time: ', self.idx_end_time)

        b1 = QGroupBox('View')
        form = QFormLayout()
        b1.setLayout(form)

        self.idx_start = QLabel('')
        self.idx_start.setToolTip('Start time in seconds from the beginning of'
                                  ' the recordings')
        self.idx_length = QLabel('')
        self.idx_length.setToolTip('Duration of the time window in seconds')
        self.idx_scaling = QLabel('')
        self.idx_scaling.setToolTip('Global scaling for all the channels')
        self.idx_distance = QLabel('')
        self.idx_distance.setToolTip('Visual distances between the traces of '
                                     'individual channels')

        form.addRow('Start Time:', self.idx_start)
        form.addRow('Length:', self.idx_length)
        form.addRow('Scaling:', self.idx_scaling)
        form.addRow('Distance:', self.idx_distance)

        layout = QVBoxLayout()
        layout.addWidget(b0)
        layout.addWidget(b1)

        self.setLayout(layout)
1346240
def open_dataset(path, **kwargs):
    # type: (str, T.Any) -> xr.Dataset
    
    if 'engine' in kwargs and kwargs['engine'] != 'cfgrib':
        raise ValueError("only engine=='cfgrib' is supported")
    kwargs['engine'] = 'cfgrib'
    return xr.backends.api.open_dataset(path, **kwargs)
1042927
def load_dataset(*args, **kwargs):
    
    warnings.warn(
        "load_dataset` will be removed in a future version of xarray. The "
        "current behavior of this function can be achived by using "
        "`tutorial.open_dataset(...).load()`.",
        DeprecationWarning, stacklevel=2)
    return open_dataset(*args, **kwargs).load()
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)