Load all samples in the dataset.
195501
def load_ratings(data_home, size):
    

    if size == '100k':
        with open(os.path.join(data_home, 'u.data'), encoding='ISO-8859-1') as f:
            lines = list(map(lambda l: list(map(int, l.rstrip().split('\t'))), f.readlines()))
    elif size == '1m':
        with open(os.path.join(data_home, 'ratings.dat'), encoding='ISO-8859-1') as f:
            lines = list(map(lambda l: list(map(int, l.rstrip().split('::'))), f.readlines()))

    ratings = []

    for l in lines:
        # Since we consider positive-only feedback setting, ratings < 5 will be excluded.
        if l[2] == 5:
            ratings.append(l)

    ratings = np.asarray(ratings)

    # sorted by timestamp
    return ratings[np.argsort(ratings[:, 3])]
813006
def _load_all(self):
        '''
        
        '''
        with self._lock:
            for name in self.file_mapping:
                if name in self.loaded_files or name in self.missing_modules:
                    continue
                self._load_module(name)

            self.loaded = True
1156671
def load_all_distributions(self):
        """"""
        distributions = []
        for this in dir(scipy.stats):
            if "fit" in eval("dir(scipy.stats." + this +")"):
                distributions.append(this)
        self.distributions = distributions[:]
1243877
def load_data(self, idx):
        """"""
        for subseqs in self:
            if isinstance(subseqs, abctools.InputSequencesABC):
                subseqs.load_data(idx)
1344306
def sample(cls, dataset, samples=[]):
        
        ndims = dataset.ndims
        dimensions = dataset.dimensions(label='name')
        arrays = [dataset.data[vdim.name] for vdim in dataset.vdims]
        data = defaultdict(list)

        for sample in samples:
            if np.isscalar(sample): sample = [sample]
            if len(sample) != ndims:
                sample = [sample[i] if i < len(sample) else None
                          for i in range(ndims)]
            sampled, int_inds = [], []
            for d, ind in zip(dimensions, sample):
                cdata = dataset.data[d]
                mask = cls.key_select_mask(dataset, cdata, ind)
                inds = np.arange(len(cdata)) if mask is None else np.argwhere(mask)
                int_inds.append(inds)
                sampled.append(cdata[mask])
            for d, arr in zip(dimensions, np.meshgrid(*sampled)):
                data[d].append(arr)
            for vdim, array in zip(dataset.vdims, arrays):
                da = dask_array_module()
                flat_index = np.ravel_multi_index(tuple(int_inds)[::-1], array.shape)
                if da and isinstance(array, da.Array):
                    data[vdim.name].append(array.flatten().vindex[tuple(flat_index)])
                else:
                    data[vdim.name].append(array.flat[flat_index])
        concatenated = {d: np.concatenate(arrays).flatten() for d, arrays in data.items()}
        return concatenated
793367
def load_data(self, pdbid):
        """"""
        f = urlopen("http://projects.biotec.tu-dresden.de/plip-rest/pdb/%s?format=xml" % pdbid.lower())
        self.doc = etree.parse(f)
1367810
def allFeatures(self):
        
        for dataset in self.getDatasets():
            for featureSet in dataset.getFeatureSets():
                for feature in featureSet.getFeatures():
                    yield feature
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
737633
def main():
    """"""
    # Read config file. This has to get updated via git
    project_root = utils.get_project_root()
    infofile = os.path.join(project_root, "raw-datasets/info.yml")
    logging.info("Read '%s'...", infofile)
    with open(infofile, 'r') as ymlfile:
        datasets = yaml.load(ymlfile)
    for dataset in datasets:
        local_path_file = os.path.join(project_root, dataset['online_path'])
        i = 0
        while not is_file_consistent(local_path_file, dataset['md5']) and i < 3:
            if os.path.isfile(local_path_file):
                local_file_size = os.path.getsize(local_path_file)
                logging.info("MD5 codes differ. ")
                logging.info("The file size of the downloaded file is %s.",
                             utils.sizeof_fmt(local_file_size))
            logging.info("Download the file '%s'...", dataset['online_path'])
            urllib.urlretrieve(dataset['url'], local_path_file)
            i += 1
        if i < 10:
            logging.info("Found '%s'.", dataset['online_path'])
385068
def run_parse(self):
        """"""
        # Data set already has source file names from load_inputs
        parsedset = {}
        parsedset['data_set'] = []
        for log in self.input_files:
            parsemodule = self.parse_modules[self.args.parser]
            try:
                if self.args.tzone:
                    parsemodule.tzone = self.args.tzone
            except NameError: pass
            parsedset['data_set'].append(parsemodule.parse_file(log))
        self.data_set = parsedset
        del(parsedset)
1171412
def load_data(self, test_percent=0.15):
        """"""
        train_images = list()
        train_labels = list()
        test_images = list()
        test_labels = list()
        return train_images, train_labels, test_images, test_labels
1337845
def add_sample(self, **data):
        """"""
        missing_dimensions = set(data).difference(self.dimensions)

        if missing_dimensions:
            raise KeyError('Dimensions not defined in this series: %s'
                           % ', '.join(missing_dimensions))

        for dim in self.dimensions:
            getattr(self, dim).append(data.get(dim))
1147046
def _create_dataset(self, *data):
        """"""
        # Make sure data is a tuple of dense tensors
        data = [self._to_torch(x, dtype=torch.FloatTensor) for x in data]
        return TensorDataset(*data)
1109652
def choose_raw_dataset(currently=""):
    """"""
    folder = os.path.join(get_project_root(), "raw-datasets")
    files = [os.path.join(folder, name) for name in os.listdir(folder)
             if name.endswith(".pickle")]
    default = -1
    for i, filename in enumerate(files):
        if os.path.basename(currently) == os.path.basename(filename):
            default = i
        if i != default:
            print("[%i]\t%s" % (i, os.path.basename(filename)))
        else:
            print("\033[1m[%i]\033[0m\t%s" % (i, os.path.basename(filename)))
    i = input_int_default("Choose a dataset by number: ", default)
    return files[i]
731388
def _load_dataset_with_area(self, dsid, coords):
        """"""
        file_handlers = self._get_file_handlers(dsid)
        if not file_handlers:
            return

        area = self._load_dataset_area(dsid, file_handlers, coords)

        try:
            ds = self._load_dataset_data(file_handlers, dsid)
        except (KeyError, ValueError) as err:
            logger.exception("Could not load dataset '%s': %s", dsid, str(err))
            return None

        if area is not None:
            ds.attrs['area'] = area
            calc_coords = (('x' not in ds.coords) or('y' not in ds.coords)) and hasattr(area, 'get_proj_vectors_dask')
            if calc_coords and hasattr(area, 'get_proj_vectors'):
                ds['x'], ds['y'] = area.get_proj_vectors()
            elif calc_coords:
                # older pyresample with dask-only method
                ds['x'], ds['y'] = area.get_proj_vectors_dask(CHUNK_SIZE)
        return ds
355120
def run(dataset):
    """"""
    config = _get_config()

    if dataset:
        _print("getting dataset from brains...")
        cprint("done", 'green')

        # check dataset cache for dataset
        # if not exists
        # r = requests.get('https://api.github.com/events', stream=True)
        # with open(filename, 'wb') as fd:
        #     for chunk in r.iter_content(chunk_size):
        #         fd.write(chunk)

    cprint('Running "%s"' % config["run"], 'green', attrs=("underline",))
    call(config["run"].split())
887184
def load_character_images(self, verbose=True):
        
        for dataset in self.character_sets:
            assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                      "rerun to try again or download datasets manually."

        for dataset in self.character_sets:
            for image, label in self.load_dataset(dataset, verbose=verbose):
                yield image, label