Break the file up into sectors
491427
function sectorify(file, ssz) {
	var nsectors = Math.ceil(file.length/ssz)-1;
	var sectors = new Array(nsectors);
	for(var i=1; i < nsectors; ++i) sectors[i-1] = file.slice(i*ssz,(i+1)*ssz);
	sectors[nsectors-1] = file.slice(nsectors*ssz);
	return sectors;
}
526044
def _split_regions(chrom, start, end):
    
    window_size = 1e5
    if end - start < window_size * 5:
        return [(chrom, start, end)]
    else:
        out = []
        for r in pybedtools.BedTool().window_maker(w=window_size,
                                                   b=pybedtools.BedTool("%s\t%s\t%s" % (chrom, start, end),
                                                                        from_string=True)):
            out.append((r.chrom, r.start, r.end))
        return out
1234473
def split_file_info(fileinfo):
    
    current_year = datetime.datetime.now().strftime('%Y')
    files = []
    for line in fileinfo:
        parts = re.split(
            r'^([\-dbclps])' +                  # Directory flag [1]
            r'([\-rwxs]{9})\s+' +               # Permissions [2]
            r'(\d+)\s+' +                       # Number of items [3]
            r'([a-zA-Z0-9_-]+)\s+' +            # File owner [4]
            r'([a-zA-Z0-9_-]+)\s+' +            # File group [5]
            r'(\d+)\s+' +                       # File size in bytes [6]
            r'(\w{3}\s+\d{1,2})\s+' +           # 3-char month and 1/2-char day of the month [7]
            r'(\d{1,2}:\d{1,2}|\d{4})\s+' +     # Time or year (need to check conditions) [+= 7]
            r'(.+)$',                           # File/directory name [8]
            line
        )

        date = parts[7]
        time = parts[8] if ':' in parts[8] else '00:00'
        year = parts[8] if ':' not in parts[8] else current_year
        dt_obj = parser.parse("%s %s %s" % (date, year, time))

        files.append({
            'directory': parts[1],
            'perms': parts[2],
            'items': parts[3],
            'owner': parts[4],
            'group': parts[5],
            'size': int(parts[6]),
            'date': date,
            'time': time,
            'year': year,
            'name': parts[9],
            'datetime': dt_obj
        })
    return files
439724
function flattenWrapFile(wrap, keyName, absFilePath) {
        var keyFileName = keyName + 'File';

        if (typeof wrap[keyName] !== 'string' && wrap[keyFileName]) {
            wrap[keyName] = '';
            if (typeof wrap[keyFileName] === 'string') {
                wrap[keyFileName] = [wrap[keyFileName]];
            }
            wrap[keyFileName].forEach(function (fileName) {
                wrap[keyName] += (wrap[keyName] ? '\n' : '') +
                    file.readFile(build.makeAbsPath(fileName, absFilePath));
            });
        } else if (typeof wrap[keyName] !== 'string') {
            throw new Error('wrap.' + keyName + ' or wrap.' + keyFileName + ' malformed');
        }
    }
696501
def split_files(files, split_train, split_val, use_test):
    
    files_train = files[:split_train]
    files_val = files[split_train:split_val] if use_test else files[split_train:]

    li = [(files_train, 'train'), (files_val, 'val')]

    # optional test folder
    if use_test:
        files_test = files[split_val:]
        li.append((files_test, 'test'))
    return li
15529
def split_vcf(in_file, ref_file, config, out_dir=None):
    
    if out_dir is None:
        out_dir = os.path.join(os.path.dirname(in_file), "split")
    out_files = []
    with open(ref.fasta_idx(ref_file, config)) as in_handle:
        for line in in_handle:
            chrom, size = line.split()[:2]
            out_file = os.path.join(out_dir,
                                    os.path.basename(replace_suffix(append_stem(in_file, "-%s" % chrom), ".vcf")))
            subset_vcf(in_file, (chrom, 0, size), out_file, config)
            out_files.append(out_file)
    return out_files
1362751
def _walk_through_splits(self):
        
        def walk(split):
            for c in split:
                if isinstance(c, (HSplit, VSplit)):
                    yield split, c
                    for i in walk(c):
                        yield i

        return walk(self.root)
971475
def from_file(cls, f):
        """"""
        ret = cls()
        section = None
        setting = None
        for lineno, line in enumerate(f.readlines()):
            line = line.lstrip()
            if setting is None:
                if _strip_comments(line).strip() == "":
                    continue
                if line[0] == "[":
                    line = _strip_comments(line).rstrip()
                    if line[-1] != "]":
                        raise ValueError("expected trailing ]")
                    key = line.strip()
                    pts = key[1:-1].split(" ", 1)
                    pts[0] = pts[0].lower()
                    if len(pts) == 2:
                        if pts[1][0] != "\"" or pts[1][-1] != "\"":
                            raise ValueError(
                                "Invalid subsection " + pts[1])
                        else:
                            pts[1] = pts[1][1:-1]
                        if not _check_section_name(pts[0]):
                            raise ValueError("invalid section name %s" %
                                             pts[0])
                        section = (pts[0], pts[1])
                    else:
                        if not _check_section_name(pts[0]):
                            raise ValueError("invalid section name %s" %
                                    pts[0])
                        pts = pts[0].split(".", 1)
                        if len(pts) == 2:
                            section = (pts[0], pts[1])
                        else:
                            section = (pts[0], )
                    ret._values[section] = {}
                else:
                    if section is None:
                        raise ValueError("setting %r without section" % line)
                    try:
                        setting, value = line.split("=", 1)
                    except ValueError:
                        setting = line
                        value = "true"
                    setting = setting.strip().lower()
                    if not _check_variable_name(setting):
                        raise ValueError("invalid variable name %s" % setting)
                    if value.endswith("\\\n"):
                        value = value[:-2]
                        continuation = True
                    else:
                        continuation = False
                    value = _parse_string(value)
                    ret._values[section][setting] = value
                    if not continuation:
                        setting = None
            else:  # continuation line
                if line.endswith("\\\n"):
                    line = line[:-2]
                    continuation = True
                else:
                    continuation = False
                value = _parse_string(line)
                ret._values[section][setting] += value
                if not continuation:
                    setting = None
        return ret
1333172
def splitroot(self, path, sep=None):
        """"""
        if sep is None:
            sep = self.filesystem.path_separator
        if self.filesystem.is_windows_fs:
            return self._splitroot_with_drive(path, sep)
        return self._splitroot_posix(path, sep)
351425
def split_subpath(path, subpath)
      return "" if path == subpath
      path = File.join(path, ''.freeze)
      if subpath.start_with?(path)
        subpath[path.length..-1]
      else
        nil
      end
    end
659445
def unpack(src_dir, dst_dir):
    ''''''
    for dirpath, dirnames, filenames in os.walk(src_dir):
        subdir = os.path.relpath(dirpath, src_dir)
        for f in filenames:
            src = os.path.join(dirpath, f)
            dst = os.path.join(dst_dir, subdir, f)
            os.renames(src, dst)
        for n, d in reversed(list(enumerate(dirnames))):
            src = os.path.join(dirpath, d)
            dst = os.path.join(dst_dir, subdir, d)
            if not os.path.exists(dst):
                # Directory does not exist in destination,
                # rename it and prune it from os.walk list.
                os.renames(src, dst)
                del dirnames[n]
    # Cleanup.
    for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True):
        assert not filenames
        os.rmdir(dirpath)
791641
def fromfile(cls, f):
        """"""
        filter = cls()
        filter._setup(*unpack(cls.FILE_FMT, f.read(calcsize(cls.FILE_FMT))))
        nfilters, = unpack(b'<l', f.read(calcsize(b'<l')))
        if nfilters > 0:
            header_fmt = b'<' + b'Q'*nfilters
            bytes = f.read(calcsize(header_fmt))
            filter_lengths = unpack(header_fmt, bytes)
            for fl in filter_lengths:
                filter.filters.append(BloomFilter.fromfile(f, fl))
        else:
            filter.filters = []

        return filter
1368425
def splitall(path):
    
    head, tail = os.path.split(os.path.normpath(path))
    components = []
    while tail:
        components.insert(0, tail)
        head, tail = os.path.split(head)
    return components
576673
def get_sectors(self, start, end=None):
        
        s = self.get_sector_slice(start, end)
        return self.bytes[s], self.style[s]
557567
function segments(filepath) {
  const parsed = path.parse(filepath);
  const dirWithoutRoot = parsed.dir.substring(parsed.root.length);

  return dirWithoutRoot
    .split(path.sep)
    .concat(parsed.name)
    .reverse();
}
543036
def splits_and_paths(self, data_dir):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    def append_epoch(paths):
      return [
          "{}.{}".format(path, self.current_epoch)
          for path in paths
      ]

    # We set shuffled=True as we don't want to shuffle on disk later.
    return [
        (split["split"], append_epoch(filepath_fns[split["split"]](
            data_dir, split["shards"], shuffled=True
        )))
        for split in self.dataset_splits
    ]
1146712
def _collapse(in_file):
    
    seqcluster = op.join(utils.get_bcbio_bin(), "seqcluster")
    out_file = "%s.fastq" % utils.splitext_plus(append_stem(in_file, "_trimmed"))[0]
    out_dir = os.path.dirname(in_file)
    if file_exists(out_file):
        return out_file
    cmd = ("{seqcluster} collapse -o {out_dir} -f {in_file} -m 1 --min_size 16")
    do.run(cmd.format(**locals()), "Running seqcluster collapse in %s." % in_file)
    return out_file