Subsets that make up each split of the dataset for the language pair.
554515
def subsets(self):
    """"""
    source, target = self.builder_config.language_pair
    filtered_subsets = {}
    for split, ss_names in self._subsets.items():
      filtered_subsets[split] = []
      for ss_name in ss_names:
        ds = DATASET_MAP[ss_name]
        if ds.target != target or source not in ds.sources:
          logging.info(
              "Skipping sub-dataset that does not include language pair: %s",
              ss_name)
        else:
          filtered_subsets[split].append(ss_name)
    logging.info("Using sub-datasets: %s", filtered_subsets)
    return filtered_subsets
408801
private static void splitInSentences(HashSet<String> sentencesSet, String text) {
		
		if (inputType == inputTypes.PARAGRAPH) { // split in multiple sentences
			LinkedList<LinkedList<Word>> sentences = Splitting.tokenizeSentences(LVMorphologyReaderAndWriter.getAnalyzer(), text);
			for (LinkedList<Word> sentence : sentences) 
			{
				if(sentence != null)
				{
					String madeSentence = "";
					for(int w = 0; w < sentence.size(); w++)
					{
						madeSentence = madeSentence + " " + sentence.get(w);
					}
					madeSentence = madeSentence.trim();
					//pārbaudam, vai teikuma atšķiribas nav punkts teikuma beigas
					if(madeSentence.length() > 0 && madeSentence.lastIndexOf(".") >= 0) madeSentence = madeSentence.substring(0, madeSentence.length() - 1);
					sentencesSet.add(madeSentence.trim());
				}
			}
		} 
	}
159741
def __parseDatasets(self):
        """"""
        datasets = []
        if self.__dataItem.has_key('dataSets'):
            for dataset in self.__dataItem['dataSets']:
                datasets.append(DataSet(Sitools2Abstract.getBaseUrl(self), dataset))
        return datasets
554018
def split(s):
    
    return map(lambda w: filter(lambda c: c != '\x00', w), lexsplit(s))
1344603
public static function split(Labeled $dataset) : self
    {
        $columns = $dataset->columns();

        $variances = array_map([Stats::class, 'variance'], $columns);

        $column = argmax($variances);

        $value = Stats::median($columns[$column]);

        $groups = $dataset->partition($column, $value);

        $min = $max = [];

        foreach ($columns as $values) {
            $min[] = min($values);
            $max[] = max($values);
        }

        return new self($column, $value, $groups, $min, $max);
    }
995243
def stash_split(fqdn, result, *argl, **argd):
    
    global _splits
    if fqdn == "sklearn.cross_validation.train_test_split":
        key = id(result[1])
        _splits[key] = result

    #We don't actually want to return anything for the analysis; we are using it
    #as a hook to save pointers to the dataset split so that we can easily
    #analyze performance later on.
    return None
254903
def data(self, data, part=False, dataset=''):
        
        return super().data(data, part)
1070459
def _get_raw_data(subset):
  """"""
  raw_data = _LOADED.get(subset)
  if raw_data is not None:
    return raw_data, _LOADED["vocab"]
  else:
    train_data, valid_data, test_data, vocab = ptb_reader.ptb_raw_data(
        FLAGS.data_path)
    _LOADED.update({
        "train": np.array(train_data),
        "valid": np.array(valid_data),
        "test": np.array(test_data),
        "vocab": vocab
    })
    return _LOADED[subset], vocab
1147942
def classify_languages(self):
        
        return BlobsWithLanguageDataFrame(self._engine_dataframe.classifyLanguages(),
                                          self._session, self._implicits)
924097
def partitioned_repertoire(self, direction, partition):
        """"""
        repertoires = [
            self.repertoire(direction, part.mechanism, part.purview)
            for part in partition
        ]
        return functools.reduce(np.multiply, repertoires)
132438
def parse(lang_sample):
    """"""
    words = words_from_archive(lang_sample, include_dups=True)
    counts = zero_default_dict()
    for word in words:
        counts[word] += 1
    return set(words), counts
584068
def get_dataset(corpora):
    
    # TODO: make filter methods for the files

    def make_posix_path(dirpath, filename):
        dirpath = posixpath.sep.join(dirpath.split(os.sep))
        return posixpath.join(dirpath, filename)

    wav_files_in_corpora = filter(
        lambda x: x.endswith('.wav'),
        sum(
            [list(map(lambda x: make_posix_path(corpus, x), os.listdir(corpus))) for corpus in corpora],
            []
        ),
    )

    dataset = {}
    for wav_file in wav_files_in_corpora:
        subjectID = os.path.split(wav_file)[-1].split('.')[0].split('_')[0]

        if subjectID in dataset:
            dataset[subjectID].append(wav_file)
        else:
            dataset[subjectID] = [wav_file]

    return dataset
1113087
private void createJobPropsForDatasets() {
    final Set<Dataset> datasetsWithProps = Sets.newHashSet();
    for (Dataset dataset : this.datasets) {
      datasetsWithProps.addAll(createJobPropsForDataset(dataset));
    }

    this.datasets.clear();
    this.datasets.addAll(datasetsWithProps);
  }
105678
def dataset_prepare(self):
        ''''''

        # Initialize the prepare subcommand's argparser
        parser = argparse.ArgumentParser(description='Preprocess a raw dialogue corpus into a dsrt dataset')
        self.init_dataset_prepare_args(parser)

        # Parse the args we got
        args = parser.parse_args(sys.argv[3:])
        args.config = ConfigurationLoader(args.config).load().data_config

        print(CLI_DIVIDER  + '\n')
        Preprocessor(**vars(args)).run()
77453
def datasets(self):
        """"""
        return {key: val['ds'] for key, val in six.iteritems(
            self._get_ds_descriptions(self.array_info(ds_description=['ds'])))}
1119924
def _split_generators(self, dl_manager):
    """"""
    # Download the full MNIST Database
    filenames = {
        "train_data": _MNIST_TRAIN_DATA_FILENAME,
        "train_labels": _MNIST_TRAIN_LABELS_FILENAME,
        "test_data": _MNIST_TEST_DATA_FILENAME,
        "test_labels": _MNIST_TEST_LABELS_FILENAME,
    }
    mnist_files = dl_manager.download_and_extract(
        {k: urllib.parse.urljoin(self.URL, v) for k, v in filenames.items()})

    # MNIST provides TRAIN and TEST splits, not a VALIDATION split, so we only
    # write the TRAIN and TEST splits to disk.
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=10,
            gen_kwargs=dict(
                num_examples=_TRAIN_EXAMPLES,
                data_path=mnist_files["train_data"],
                label_path=mnist_files["train_labels"],
            )),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=1,
            gen_kwargs=dict(
                num_examples=_TEST_EXAMPLES,
                data_path=mnist_files["test_data"],
                label_path=mnist_files["test_labels"],
            )),
    ]
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs