Split data into train, valid and test groups
432170
def make_data_splits(self, max_samples, valid_story=None, test_story=None):
        """"""
        # TODO Make this also work with wordlists.
        if valid_story or test_story:
            if not (valid_story and test_story):
                raise PersephoneException(
                    "We need a valid story if we specify a test story "
                    "and vice versa. This shouldn't be required but for "
                    "now it is.")

            train, valid, test = make_story_splits(valid_story, test_story,
                                                   max_samples,
                                                   self.label_type,
                                                   tgt_dir=str(self.tgt_dir))
        else:
            train, valid, test = make_data_splits(self.label_type,
                                                  train_rec_type=self.train_rec_type,
                                                  max_samples=max_samples,
                                                  tgt_dir=str(self.tgt_dir))
        self.train_prefixes = train
        self.valid_prefixes = valid
        self.test_prefixes = test
751603
def _split_batches(self, data, batch_size):
        """"""
        for i in range(0, len(data), batch_size):
            yield data[i : i + batch_size]
662804
def validate_split_runs_file(split_runs_file):
    """"""
    try:
        content = [l.strip() for l in split_runs_file.readlines()]
        if content[0].upper().split('\t') == ['NAME', 'RUN_ID']:
            return {c.split('\t')[1]: c.split('\t')[0] for c in content[1:] if c}
        else:
            sys.exit("ERROR: Mandatory header of --split_runs tsv file not found: 'NAME', 'RUN_ID'")
            logging.error("Mandatory header of --split_runs tsv file not found: 'NAME', 'RUN_ID'")
    except IndexError:
        sys.exit("ERROR: Format of --split_runs tab separated file not as expected")
        logging.error("ERROR: Format of --split_runs tab separated file not as expected")
1105044
def split_by_idxs(self, train_idx, valid_idx):
        ""
        return self.split_by_list(self[train_idx], self[valid_idx])
280816
def make_dataloader(data_train, data_val, data_test, args,
                    use_average_length=False, num_shards=0, num_workers=8):
    """"""
    data_train_lengths = get_data_lengths(data_train)
    data_val_lengths = get_data_lengths(data_val)
    data_test_lengths = get_data_lengths(data_test)
    train_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
                                  btf.Stack(dtype='float32'), btf.Stack(dtype='float32'))
    test_batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(),
                                 btf.Stack(dtype='float32'), btf.Stack(dtype='float32'),
                                 btf.Stack())
    target_val_lengths = list(map(lambda x: x[-1], data_val_lengths))
    target_test_lengths = list(map(lambda x: x[-1], data_test_lengths))
    if args.bucket_scheme == 'constant':
        bucket_scheme = nlp.data.ConstWidthBucket()
    elif args.bucket_scheme == 'linear':
        bucket_scheme = nlp.data.LinearWidthBucket()
    elif args.bucket_scheme == 'exp':
        bucket_scheme = nlp.data.ExpWidthBucket(bucket_len_step=1.2)
    else:
        raise NotImplementedError
    train_batch_sampler = nlp.data.FixedBucketSampler(lengths=data_train_lengths,
                                                      batch_size=args.batch_size,
                                                      num_buckets=args.num_buckets,
                                                      ratio=args.bucket_ratio,
                                                      shuffle=True,
                                                      use_average_length=use_average_length,
                                                      num_shards=num_shards,
                                                      bucket_scheme=bucket_scheme)
    logging.info('Train Batch Sampler:\n%s', train_batch_sampler.stats())
    train_data_loader = nlp.data.ShardedDataLoader(data_train,
                                                   batch_sampler=train_batch_sampler,
                                                   batchify_fn=train_batchify_fn,
                                                   num_workers=num_workers)

    val_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_val_lengths,
                                                    batch_size=args.test_batch_size,
                                                    num_buckets=args.num_buckets,
                                                    ratio=args.bucket_ratio,
                                                    shuffle=False,
                                                    use_average_length=use_average_length,
                                                    bucket_scheme=bucket_scheme)
    logging.info('Valid Batch Sampler:\n%s', val_batch_sampler.stats())
    val_data_loader = gluon.data.DataLoader(data_val,
                                            batch_sampler=val_batch_sampler,
                                            batchify_fn=test_batchify_fn,
                                            num_workers=num_workers)
    test_batch_sampler = nlp.data.FixedBucketSampler(lengths=target_test_lengths,
                                                     batch_size=args.test_batch_size,
                                                     num_buckets=args.num_buckets,
                                                     ratio=args.bucket_ratio,
                                                     shuffle=False,
                                                     use_average_length=use_average_length,
                                                     bucket_scheme=bucket_scheme)
    logging.info('Test Batch Sampler:\n%s', test_batch_sampler.stats())
    test_data_loader = gluon.data.DataLoader(data_test,
                                             batch_sampler=test_batch_sampler,
                                             batchify_fn=test_batchify_fn,
                                             num_workers=num_workers)
    return train_data_loader, val_data_loader, test_data_loader
1209811
def load_data_split(proc_data_dir):
    
    ds_train = Dataset.load(path.join(proc_data_dir, 'train.bin'))
    ds_val = Dataset.load(path.join(proc_data_dir, 'val.bin'))
    ds_test = Dataset.load(path.join(proc_data_dir, 'test.bin'))
    return ds_train, ds_val, ds_test
13322
def gatk_splitreads(data):
    
    broad_runner = broad.runner_from_config(dd.get_config(data))
    ref_file = dd.get_ref_file(data)
    deduped_bam = dd.get_deduped_bam(data)
    base, ext = os.path.splitext(deduped_bam)
    split_bam = base + ".splitN" + ext
    if file_exists(split_bam):
        data = dd.set_split_bam(data, split_bam)
        return data
    gatk_type = broad_runner.gatk_type()
    with file_transaction(data, split_bam) as tx_split_bam:
        params = ["-T", "SplitNCigarReads",
                  "-R", ref_file,
                  "-I", deduped_bam]
        if gatk_type == "gatk4":
            params += ["--output", tx_split_bam]
        else:
            params += ["-rf", "ReassignOneMappingQuality",
                       "-RMQF", "255",
                       "-RMQT", "60",
                       "-rf", "UnmappedRead",
                       "-U", "ALLOW_N_CIGAR_READS",
                       "-o", tx_split_bam]
            if dd.get_quality_format(data) == "illumina":
                params += ["--fix_misencoded_quality_scores", "-fixMisencodedQuals"]
        broad_runner.run_gatk(params)
    bam.index(split_bam, dd.get_config(data))
    data = dd.set_split_bam(data, split_bam)
    return data
943220
def split_data(data, squeeze=False):
    
    vdata = np.atleast_2d(data)
    nr_freqs = int(vdata.shape[1] / 2)
    part1 = vdata[:, 0:nr_freqs]
    part2 = vdata[:, nr_freqs:]
    if(squeeze):
        part1 = part1.squeeze()
        part2 = part2.squeeze()
    return part1, part2
439388
def split(self, data):
		
		line = deepcopy(data)
		line_width = (self.width() - 1)

		lines = []
		while len(line):

			new_line = line[:line_width]

			new_line_pos = new_line.find('\n')
			if new_line_pos >= 0:
				new_line = line[:new_line_pos]
				line = line[(new_line_pos + 1):]
			else:
				line = line[line_width:]

			lines.append(new_line)

		return lines
1287640
def from_data(cls, data):
        
        if len(data) == 0:
            raise ValueError("data array is empty.")
        dim_x, dim_y = len(data[0][0]), len(data[0][1])
        dataset = cls(dim_x, dim_y)
        for x, y in data:
            assert len(x) == dim_x and len(y) == dim_y
            dataset.add_xy(x, y)
        return dataset
1109387
def _split_generators(self, dl_manager):
    """"""
    root_url = "http://images.cocodataset.org/"
    urls = {
        # Train/validation set
        "train_images": "zips/train2014.zip",
        "val_images": "zips/val2014.zip",
        "trainval_annotations": "annotations/annotations_trainval2014.zip",
        # Testing set (no annotations) (2014)
        "test_images": "zips/test2014.zip",
        "test_annotations": "annotations/image_info_test2014.zip",
        # Testing set (no annotations) (2015)
        "test2015_images": "zips/test2015.zip",
        "test2015_annotations": "annotations/image_info_test2015.zip",
    }
    extracted_paths = dl_manager.download_and_extract({
        key: root_url + url for key, url in urls.items()
    })

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=10,
            gen_kwargs=dict(
                image_dir=extracted_paths["train_images"],
                annotation_dir=extracted_paths["trainval_annotations"],
                split_type="train2014",
            )),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            num_shards=10,
            gen_kwargs=dict(
                image_dir=extracted_paths["val_images"],
                annotation_dir=extracted_paths["trainval_annotations"],
                split_type="val2014",
            )),
        # Warning: Testing split only contains the images without any annotation
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=10,
            gen_kwargs=dict(
                image_dir=extracted_paths["test_images"],
                annotation_dir=extracted_paths["test_annotations"],
                split_type="test2014",
                has_annotation=False,
            )),
        tfds.core.SplitGenerator(
            name="test2015",
            num_shards=10,
            gen_kwargs=dict(
                image_dir=extracted_paths["test2015_images"],
                annotation_dir=extracted_paths["test2015_annotations"],
                split_type="test2015",
                has_annotation=False,
            )),
    ]
132697
def split_data(self):
        

        try:
            # Extract data ranging in time_period1
            time_period1 = (slice(self.baseline_period[0], self.baseline_period[1]))
            self.baseline_in = self.original_data.loc[time_period1, self.input_col]
            self.baseline_out = self.original_data.loc[time_period1, self.output_col]

            if self.exclude_time_period:
                for i in range(0, len(self.exclude_time_period), 2):
                    # Drop data ranging in exclude_time_period1
                    exclude_time_period1 = (slice(self.exclude_time_period[i], self.exclude_time_period[i+1]))
                    self.baseline_in.drop(self.baseline_in.loc[exclude_time_period1].index, axis=0, inplace=True)
                    self.baseline_out.drop(self.baseline_out.loc[exclude_time_period1].index, axis=0, inplace=True)
        except Exception as e:
            raise e

        # CHECK: Can optimize this part
        # Error checking to ensure time_period values are valid
        if self.projection_period:
            for i in range(0, len(self.projection_period), 2):
                period = (slice(self.projection_period[i], self.projection_period[i+1]))
                try:
                    self.original_data.loc[period, self.input_col]
                    self.original_data.loc[period, self.output_col]
                except Exception as e:
                    raise e
384813
def train(self, x_data, y_data):
        
        x_train, _, y_train, _ = train_test_split(
            x_data,
            y_data,
            test_size=0.67,
            random_state=None
        )  # cross-split

        self.model.fit(x_train, y_train)
696501
def split_files(files, split_train, split_val, use_test):
    
    files_train = files[:split_train]
    files_val = files[split_train:split_val] if use_test else files[split_train:]

    li = [(files_train, 'train'), (files_val, 'val')]

    # optional test folder
    if use_test:
        files_test = files[split_val:]
        li.append((files_test, 'test'))
    return li
1311355
def random_split(valid_pct:float, *arrs:NPArrayableList)->SplitArrayList:
    ""
    assert (valid_pct>=0 and valid_pct<=1), 'Validation set percentage should be between 0 and 1'
    is_train = np.random.uniform(size=(len(arrs[0]),)) > valid_pct
    return arrays_split(is_train, *arrs)
1045158
def split_variants_by_sample(data):
    
    # not split, do nothing
    if "group_orig" not in data:
        return [[data]]
    # cancer tumor/normal
    elif (vcfutils.get_paired_phenotype(data)
            and "tumor" in [vcfutils.get_paired_phenotype(d) for d in get_orig_items(data)]):
        out = []
        for i, sub_data in enumerate(get_orig_items(data)):
            if vcfutils.get_paired_phenotype(sub_data) == "tumor":
                cur_batch = tz.get_in(["metadata", "batch"], data)
                if cur_batch:
                    sub_data["metadata"]["batch"] = cur_batch
                sub_data["vrn_file"] = data["vrn_file"]
            else:
                sub_data.pop("vrn_file", None)
            out.append([sub_data])
        return out
    # joint calling or population runs, do not split back up and keep in batches
    else:
        out = []
        for sub_data in get_orig_items(data):
            cur_batch = tz.get_in(["metadata", "batch"], data)
            if cur_batch:
                sub_data["metadata"]["batch"] = cur_batch
            sub_data["vrn_file_batch"] = data["vrn_file"]
            sub_data["vrn_file"] = data["vrn_file"]
            out.append([sub_data])
        return out
1065178
def __validate(data, classes, labels):
        ""

        if not isinstance(data, dict):
            raise TypeError(
                'data must be a dict! keys: sample ID or any unique identifier')
        if not isinstance(labels, dict):
            raise TypeError(
                'labels must be a dict! keys: sample ID or any unique identifier')
        if classes is not None:
            if not isinstance(classes, dict):
                raise TypeError(
                    'labels must be a dict! keys: sample ID or any unique identifier')

        if not len(data) == len(labels) == len(classes):
            raise ValueError('Lengths of data, labels and classes do not match!')
        if not set(list(data)) == set(list(labels)) == set(list(classes)):
            raise ValueError(
                'data, classes and labels dictionaries must have the same keys!')

        num_features_in_elements = np.unique([sample.size for sample in data.values()])
        if len(num_features_in_elements) > 1:
            raise ValueError(
                'different samples have different number of features - invalid!')

        return True