Group files that match by globs for merging, rather than by explicit pairs.
192750
def _find_glob_matches(in_files, metadata):
    
    reg_files = copy.deepcopy(in_files)
    glob_files = []
    for glob_search in [x for x in metadata.keys() if "*" in x]:
        cur = []
        for fname in in_files:
            if fnmatch.fnmatch(fname, "*/%s" % glob_search):
                cur.append(fname)
                reg_files.remove(fname)
        assert cur, "Did not find file matches for %s" % glob_search
        glob_files.append(cur)
    return reg_files, glob_files
898639
def _glob(pattern, skip_dirs=False):
    """"""
    # Implicit recursive iff no / in pattern and starts with *
    if "/" not in pattern and pattern.startswith("*"):
        files = glob.glob(f"**/{pattern}", recursive=True)
    else:
        files = glob.glob(pattern, recursive=True)

    # Expand dirs
    all_files = set()
    for file in files:
        if os.path.isdir(file) and not skip_dirs:
            all_files.update(set(f for f in _glob(f"{file}/**/*", skip_dirs=True) if not os.path.isdir(f)))
        else:
            all_files.add(file)

    # Normalize all files
    return {str(Path(f)) for f in all_files}
758721
def _combine_ngrams(ngrams, joiner) -> str:
        """"""
        if isinstance(ngrams, str):
            return ngrams
        else:
            combined = joiner.join(ngrams)
            return combined
714849
def fiGroupFromLinkageGroup(matchArr, arrPos, groupId, timeKey, massKey):
    
    fgi = Fgi(groupId)
    matchArr['isAnnotated'][arrPos]

    minT = numpy.min(matchArr[timeKey][arrPos])
    maxT = numpy.max(matchArr[timeKey][arrPos])
    minM = numpy.min(matchArr[massKey][arrPos])
    maxM = numpy.max(matchArr[massKey][arrPos])

    fgi.isValid = True
    fgi.specfiles = matchArr['specfile'][arrPos].tolist()
    fgi.featureIds = matchArr['id'][arrPos].tolist()
    fgi.isAnnotated = numpy.any(matchArr['isAnnotated'][arrPos])
    fgi.coordinates = ((minT, maxT), (minM, maxM))
    #fgi.clusterType = clusterType

    return fgi
152168
def join(input_files, output_file):
    '''
    
    '''

    # get feature collections
    final_features = []
    for file in input_files:
        with open(file) as f:
            feat_collection = geojson.load(f)
            final_features += feat_collection['features']

    feat_collection['features'] = final_features

    # write to output file
    with open(output_file, 'w') as f:
        geojson.dump(feat_collection, f)
758838
def globlookup(pattern, root):
    
    for subdir, dirnames, filenames in os.walk(root):
        d = subdir[len(root) + 1:]
        files = (os.path.join(d, f) for f in filenames)
        for f in fnmatch.filter(files, pattern):
            yield f
953616
def glob_files_locally(folder_path, pattern):
    """"""

    pattern = os.path.join(
        folder_path, pattern.lstrip('/')) if pattern else None

    len_folder_path = len(folder_path) + 1
    for root, _, files in os.walk(folder_path):
        for f in files:
            full_path = os.path.join(root, f)
            if not pattern or _match_path(full_path, pattern):
                yield (full_path, full_path[len_folder_path:])
1104467
def globfilter(filenames, patterns, *, flags=0):
    """"""

    matches = []

    flags = _flag_transform(flags)
    unix = _wcparse.is_unix_style(flags)
    obj = _wcparse.compile(_wcparse.split(patterns, flags), flags)

    for filename in filenames:
        if not unix:
            filename = util.norm_slash(filename)
        if obj.match(filename):
            matches.append(filename)
    return matches
827012
def merge(bedfiles):
    
    if isinstance(bedfiles, list):
        catted = concat(bedfiles)
    else:
        catted = concat([bedfiles])
    if catted:
        return concat(bedfiles).sort().merge()
    else:
        return catted
976341
function match (filename, patterns, ignorePatterns) {
  function isMatch (ps) {
    return ps.some(p => {
      return mm.isMatch(filename, p, mmOptions)
    })
  }

  let isIgnore = false

  if (Array.isArray(ignorePatterns) && ignorePatterns.length) {
    isIgnore = isMatch(ignorePatterns)
  }

  if (isIgnore) {
    return false
  }

  return isMatch(patterns)
}
1166050
def merge(args):
    
    p = OptionParser(merge.__doc__)
    p.set_outdir(outdir="outdir")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    folders = args
    outdir = opts.outdir
    mkdir(outdir)

    files = flatten(glob("{0}/*.*.fastq".format(x)) for x in folders)
    files = list(files)
    key = lambda x: op.basename(x).split(".")[0]
    files.sort(key=key)
    for id, fns in groupby(files, key=key):
        fns = list(fns)
        outfile = op.join(outdir, "{0}.fastq".format(id))
        FileMerger(fns, outfile=outfile).merge(checkexists=True)
850327
def mergeclean(args):
    
    from itertools import groupby
    from jcvi.formats.base import FileShredder

    p = OptionParser(mergeclean.__doc__)
    p.set_sep(sep="_", help="Separator to group per prefix")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    files = sorted(args)
    sep = opts.sep
    key = lambda x: x.split(sep)[0]
    mtime = lambda x: os.stat(x).st_mtime
    for pf, fs in groupby(files, key=key):
        fs = list(fs)
        if len(fs) == 1:
            continue
        newest_f = max(fs, key=mtime)
        print("|".join(fs), "=>", newest_f, file=sys.stderr)
        fs.remove(newest_f)
        FileShredder(fs)
71732
def glob(*args):
    
    if len(args) is 1 and isinstance(args[0], list):
        args = args[0]
    matches = []
    for pattern in args:
        for item in glob2.glob(pattern):
            if not os.path.isdir(item):
                matches.append(item)
    return matches
1290636
def match_files(self, matched, unmatched):
        """"""
        this_match = set(self.file_filter(unmatched))
        matched |= this_match
        unmatched -= this_match
177408
def merge_extras(items, config):
    
    final = {}
    for extra_name in items[0]["disambiguate"].keys():
        in_files = []
        for data in items:
            in_files.append(data["disambiguate"][extra_name])
        out_file = "%s-allmerged%s" % os.path.splitext(in_files[0])
        if in_files[0].endswith(".bam"):
            merged_file = merge.merge_bam_files(in_files, os.path.dirname(out_file), items[0],
                                                out_file=out_file)
        else:
            assert extra_name == "summary", extra_name
            merged_file = _merge_summary(in_files, out_file, items[0])
        final[extra_name] = merged_file
    out = []
    for data in items:
        data["disambiguate"] = final
        out.append([data])
    return out
1124555
def match_files(files, pattern: Pattern):
    """"""

    for name in files:
        if re.match(pattern, name):
            yield name
1227954
def prt_outfiles_grouped(self, outfiles):
        """"""
        for outfile in outfiles:
            if outfile.endswith(".xlsx"):
                self.wr_xlsx(outfile)
            elif outfile.endswith(".txt"):
                self.wr_txt(outfile)
            else:
                self.wr_tsv(outfile)