read the data set
331381
private ExampleSet readDataSet(File in) throws IOException {
        logger.info("read the example set");

        //
        logger.info("read data set");
        ExampleSet inputSet = new SentenceSetCopy();
        inputSet.read(new BufferedReader(new FileReader(in)));

        return inputSet;
    }
98254
def _read(self):
        """"""
        _, data_img_name, label_img_name = self.f.readline().strip('\n').split("\t")
        data = {}
        label = {}
        data[self.data_name], label[self.label_name] = self._read_img(data_img_name, label_img_name)
        return list(data.items()), list(label.items())
159741
def __parseDatasets(self):
        """"""
        datasets = []
        if self.__dataItem.has_key('dataSets'):
            for dataset in self.__dataItem['dataSets']:
                datasets.append(DataSet(Sitools2Abstract.getBaseUrl(self), dataset))
        return datasets
254903
def data(self, data, part=False, dataset=''):
        
        return super().data(data, part)
1160255
def get_dataset(self, key, info):
        """"""

        angles = self._get_coarse_dataset(key, info)
        if angles is None:
            return

        # Fill gaps at edges of swath
        darr = DataArray(angles, dims=['y', 'x'])
        darr = darr.bfill('x')
        darr = darr.ffill('x')
        angles = darr.data

        res = self.interpolate_angles(angles, key.resolution)

        proj = DataArray(res, dims=['y', 'x'])
        proj.attrs = info.copy()
        proj.attrs['units'] = 'degrees'
        proj.attrs['platform_name'] = self.platform_name
        return proj
25826
public DataSet getDataSet(String fileName) {
        if (dataSetCache.containsKey(fileName)) {
            return dataSetCache.get(fileName);
        }
        DataSet dataSet;
        try {
            dataSet = loadDataSetAsStream(fileName);
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
        dataSetCache.put(fileName, dataSet);
        return dataSet;
    }
815567
def get_dataset(self, key, info):
        """"""
        res = super(HRITGOMSFileHandler, self).get_dataset(key, info)

        res = self.calibrate(res, key.calibration)
        res.attrs['units'] = info['units']
        res.attrs['standard_name'] = info['standard_name']
        res.attrs['wavelength'] = info['wavelength']
        res.attrs['platform_name'] = self.platform_name
        res.attrs['sensor'] = 'msu-gs'
        res.attrs['satellite_longitude'] = self.mda['projection_parameters']['SSP_longitude']
        res.attrs['satellite_latitude'] = 0
        res.attrs['satellite_altitude'] = 35785831.00

        return res
1345330
def _readData(self, id3, data):
        """"""

        for reader in self._framespec:
            if len(data) or reader.handle_nodata:
                try:
                    value, data = reader.read(id3, self, data)
                except SpecError as e:
                    raise ID3JunkFrameError(e)
            else:
                raise ID3JunkFrameError("no data left")
            self._setattr(reader.name, value)

        for reader in self._optionalspec:
            if len(data) or reader.handle_nodata:
                try:
                    value, data = reader.read(id3, self, data)
                except SpecError as e:
                    raise ID3JunkFrameError(e)
            else:
                break
            self._setattr(reader.name, value)

        return data
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
75805
def get_dataset(self, dataset_id, ds_info):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        valid_min, valid_max = self[var_path + '/attr/valid_range']
        fill_value = self.get(var_path + '/attr/_FillValue')

        d_tmp = self[var_path]
        if "index" in ds_info:
            d_tmp = d_tmp[int(ds_info["index"])]
        if "pressure_index" in ds_info:
            d_tmp = d_tmp[..., int(ds_info["pressure_index"])]
            # this is a pressure based field
            # include surface_pressure as metadata
            sp = self['Surface_Pressure']
            if 'surface_pressure' in ds_info:
                ds_info['surface_pressure'] = xr.concat((ds_info['surface_pressure'], sp))
            else:
                ds_info['surface_pressure'] = sp
            # include all the pressure levels
            ds_info.setdefault('pressure_levels', self['Pressure'][0])
        data = d_tmp

        if valid_min is not None and valid_max is not None:
            # the original .cfg/INI based reader only checked valid_max
            data = data.where((data <= valid_max))  # | (data >= valid_min))
        if fill_value is not None:
            data = data.where(data != fill_value)

        data.attrs.update(metadata)
        return data
390041
def _read_data(prefix, batch_size, augmentation=None):
    
    prefix = prefix.replace("\\", "/")
    folder = "/".join(prefix.split("/")[:-1])
    phase = prefix.split("/")[-1]
    config = json.load(open(prefix + '_config.json'))
    num_threads = config["num_threads"]

    filenames = [folder + "/" + f for f in listdir(folder) if isfile(join(folder, f)) and phase in f and not "config.json" in f]

    dataset = tf.data.TFRecordDataset(filenames=filenames, num_parallel_reads=num_threads)
    dataset = dataset.shuffle(buffer_size=10 * batch_size)
    dataset = dataset.repeat()
    dataset = dataset.map(map_func=_create_parser_fn(config, phase), num_parallel_calls=num_threads)
    if augmentation is not None:
        dataset = dataset.map(map_func=augmentation, num_parallel_calls=num_threads)
    dataset = dataset.batch(batch_size=batch_size)
    dataset = dataset.prefetch(buffer_size=1)

    return dataset
369759
def load_dataset(self, dataset, verbose=True):
        
        assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                  "rerun to try again or download datasets manually."

        if verbose:
            print("Loading %s" % dataset)

        dataset_path = self.base_dataset_path + dataset
        for path in tqdm(glob.glob(dataset_path + "/*.gnt")):
            for image, label in self.load_gnt_file(path):
                yield image, label
713029
def read(self, frame, data):
        ""
        seq = []
        while data:
            elem, data = self.spec.read(frame, data)
            seq.append(elem)
        return seq, data
1044071
def get_data(self):
		""
		array = ctypes.POINTER(ctypes.c_char * len(self))
		return ctypes.cast(self.data, array).contents.raw
343264
def dataset(self):
        """"""
        data = tablib.Dataset()
        data.headers = self.keys()

        row = _reduce_datetimes(self.values())
        data.append(row)

        return data
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs