recursively concat a set of BED files, returning a
    sorted bedtools object of the result
542702
def concat(bed_files, catted=None):
    
    bed_files = [x for x in bed_files if x]
    if len(bed_files) == 0:
        if catted:
            # move to a .bed extension for downstream tools if not already
            sorted_bed = catted.sort()
            if not sorted_bed.fn.endswith(".bed"):
                return sorted_bed.moveto(sorted_bed.fn + ".bed")
            else:
                return sorted_bed
        else:
            return catted

    if not catted:
        bed_files = list(bed_files)
        catted = bt.BedTool(bed_files.pop())
    else:
        catted = catted.cat(bed_files.pop(), postmerge=False,
                            force_truncate=False)

    return concat(bed_files, catted)
1185824
def _remove_regions(in_file, remove_beds, ext, data):
    
    from bcbio.variation import bedutils
    out_file = "%s-%s.bed" % (utils.splitext_plus(in_file)[0], ext)
    if not utils.file_uptodate(out_file, in_file):
        with file_transaction(data, out_file) as tx_out_file:
            with bedtools_tmpdir(data):
                if len(remove_beds) == 0:
                    to_remove = None
                elif len(remove_beds) == 1:
                    to_remove = remove_beds[0]
                else:
                    to_remove = "%s-all.bed" % utils.splitext_plus(tx_out_file)[0]
                    with open(to_remove, "w") as out_handle:
                        for b in remove_beds:
                            with utils.open_gzipsafe(b) as in_handle:
                                for line in in_handle:
                                    parts = line.split("\t")
                                    out_handle.write("\t".join(parts[:4]).rstrip() + "\n")
                    if utils.file_exists(to_remove):
                        to_remove = bedutils.sort_merge(to_remove, data)
                if to_remove and utils.file_exists(to_remove):
                    cmd = "bedtools subtract -nonamecheck -a {in_file} -b {to_remove} > {tx_out_file}"
                    do.run(cmd.format(**locals()), "Remove problematic regions: %s" % ext)
                else:
                    utils.symlink_plus(in_file, out_file)
    return out_file
850327
def mergeclean(args):
    
    from itertools import groupby
    from jcvi.formats.base import FileShredder

    p = OptionParser(mergeclean.__doc__)
    p.set_sep(sep="_", help="Separator to group per prefix")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    files = sorted(args)
    sep = opts.sep
    key = lambda x: x.split(sep)[0]
    mtime = lambda x: os.stat(x).st_mtime
    for pf, fs in groupby(files, key=key):
        fs = list(fs)
        if len(fs) == 1:
            continue
        newest_f = max(fs, key=mtime)
        print("|".join(fs), "=>", newest_f, file=sys.stderr)
        fs.remove(newest_f)
        FileShredder(fs)
160499
def condense(args):
    
    from itertools import groupby
    from jcvi.assembly.patch import merge_ranges

    p = OptionParser(condense.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    bedfile, = args
    bed = Bed(bedfile, sorted=False)
    key = lambda x: (x.seqid, x.start, x.end)
    for k, sb in groupby(bed, key=key):
        sb = list(sb)
        b = sb[0]
        chr, start, end, strand = merge_ranges(sb)

        id = "{0}:{1}-{2}".format(chr, start, end)
        b.accn = id
        print(b)
898639
def _glob(pattern, skip_dirs=False):
    """"""
    # Implicit recursive iff no / in pattern and starts with *
    if "/" not in pattern and pattern.startswith("*"):
        files = glob.glob(f"**/{pattern}", recursive=True)
    else:
        files = glob.glob(pattern, recursive=True)

    # Expand dirs
    all_files = set()
    for file in files:
        if os.path.isdir(file) and not skip_dirs:
            all_files.update(set(f for f in _glob(f"{file}/**/*", skip_dirs=True) if not os.path.isdir(f)))
        else:
            all_files.add(file)

    # Normalize all files
    return {str(Path(f)) for f in all_files}
1353914
def batch(items, size):
    """"""
    return [items[x:x + size] for x in xrange(0, len(items), size)]
1127384
def decomment(bed_file, out_file):
    
    if file_exists(out_file):
        return out_file

    with utils.open_gzipsafe(bed_file) as in_handle, open(out_file, "w") as out_handle:
        for line in in_handle:
            if line.startswith("#") or line.startswith("browser") or line.startswith("track"):
                continue
            else:
                out_handle.write(line)
    return out_file
11993
def merge(args):
    
    p = OptionParser(merge.__doc__)
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    bedfiles = args
    fw = must_open(opts.outfile, "w")
    for bedfile in bedfiles:
        bed = Bed(bedfile)
        pf = op.basename(bedfile).split(".")[0]
        for b in bed:
            b.seqid = "_".join((pf, b.seqid))
            print(b, file=fw)
872010
function concatFiles() {
	return Promise.all(argv._.map(handleGlob)).then(globs => {
		const files = globs.reduce((acc, cur) => acc.concat(cur), []);

		if (
			(files.length < 2 && typeof argv.banner === 'undefined' && !argv.footer) ||
			(files.length === 0 && (typeof argv.banner === 'undefined' || !argv.footer))
		) {
			throw new Error(
				chalk.bold.red('Require at least 2 file, banner or footer to concatenate. ("ncat --help" for help)\n')
			);
		}
		return files.forEach(file => {
			concat.add(file.file, file.content, file.map);
		});
	});
}
1126185
def glob(cls, files=None):
        '''
        
        '''
        files = files or []
        if isinstance(files, str):
            files = os.path.normpath(files)
            matches = lambda path: matches_patterns(path, [files])
            return [path for path in cls.get_static_files() if matches(path)]
        elif isinstance(files, (list, tuple)):
            all_files = cls.get_static_files()
            files = [os.path.normpath(f) for f in files]
            sorted_result = []
            for pattern in files:
                sorted_result.extend([f for f in all_files if matches_patterns(f, [pattern])])
            return sorted_result
1106861
def makebunches(data, commdct):
    """"""
    bunchdt = {}
    ddtt, dtls = data.dt, data.dtls
    for obj_i, key in enumerate(dtls):
        key = key.upper()
        bunchdt[key] = []
        objs = ddtt[key]
        for obj in objs:
            bobj = makeabunch(commdct, obj, obj_i)
            bunchdt[key].append(bobj)
    return bunchdt
217588
def bed(args):
    
    p = OptionParser(bed.__doc__)
    p.add_option("--switch", default=False, action="store_true",
                 help="Switch reference and aligned map elements [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    mapout, = args
    pf = mapout.split(".")[0]
    mapbed = pf + ".bed"
    bm = BinMap(mapout)
    bm.print_to_bed(mapbed, switch=opts.switch)

    return mapbed
432549
def _get_contigs_to_keep(self, filename):
        ''''''
        if filename is None:
            return set()

        with open(filename) as f:
            return {line.rstrip() for line in f}
1164013
def _bed_to_bed6(orig_file, out_dir):
    
    bed6_file = os.path.join(out_dir, "%s-bed6%s" % os.path.splitext(os.path.basename(orig_file)))
    if not utils.file_exists(bed6_file):
        with open(bed6_file, "w") as out_handle:
            for i, region in enumerate(list(x) for x in pybedtools.BedTool(orig_file)):
                region = [x for x in list(region) if x]
                fillers = [str(i), "1.0", "+"]
                full = region + fillers[:6 - len(region)]
                out_handle.write("\t".join(full) + "\n")
    return bed6_file
1245485
def concat_generator(filename, up_threshold, low_threshold=10):
  """"""
  txt = ""
  for line in tf.gfile.Open(filename):
    line = line.strip()
    if len(txt) + len(line) + 1 >= up_threshold:
      ret = txt
      txt = ""
      # We don't yield very short long parts to prevent noisy examples.
      if len(ret) > low_threshold and len(ret) < up_threshold:
        yield {"targets": ret}

    if not txt:
      txt = line
    else:
      txt = " ".join([txt, line])
1300298
def _auto_combine(datasets, concat_dims, compat, data_vars, coords,
                  infer_order_from_coords, ids):
    

    # Arrange datasets for concatenation
    if infer_order_from_coords:
        raise NotImplementedError
        # TODO Use coordinates to determine tile_ID for each dataset in N-D
        # Ignore how they were ordered previously
        # Should look like:
        # combined_ids, concat_dims = _infer_tile_ids_from_coords(datasets,
        # concat_dims)
    else:
        # Use information from the shape of the user input
        if not ids:
            # Determine tile_IDs by structure of input in N-D
            # (i.e. ordering in list-of-lists)
            combined_ids, concat_dims = _infer_concat_order_from_positions(
                datasets, concat_dims)
        else:
            # Already sorted so just use the ids already passed
            combined_ids = OrderedDict(zip(ids, datasets))

    # Check that the inferred shape is combinable
    _check_shape_tile_ids(combined_ids)

    # Repeatedly concatenate then merge along each dimension
    combined = _combine_nd(combined_ids, concat_dims, compat=compat,
                           data_vars=data_vars, coords=coords)
    return combined
497204
def merge_bed_by_name(bt):
    
    name_lines = dict()
    for r in bt:
        name = r.name
        name_lines[name] = name_lines.get(name, []) + [[r.chrom, r.start, 
                                                         r.end, r.name,
                                                         r.strand]]
    new_lines = []
    for name in name_lines.keys():
        new_lines += _merge_interval_list(name_lines[name])
    new_lines = ['\t'.join(map(str, x)) for x in new_lines]
    return pbt.BedTool('\n'.join(new_lines) + '\n', from_string=True)