Combine output details for parallelization.

    file_key is the key name of the output file used in merging. We extract
    this file from the output data.

    combine_arg_keys are extra items to pass along to the combine function.
327823
def _organize_output(output, combine_map, file_key, combine_arg_keys):
    
    out_map = collections.defaultdict(list)
    extra_args = collections.defaultdict(list)
    final_args = collections.OrderedDict()
    extras = []
    for data in output:
        cur_file = data.get(file_key)
        if not cur_file:
            extras.append([data])
        else:
            cur_out = combine_map[cur_file]
            out_map[cur_out].append(cur_file)
            extra_args[cur_out].append([data[x] for x in combine_arg_keys])
            data[file_key] = cur_out
            if cur_out not in final_args:
                final_args[cur_out] = [data]
            else:
                extras.append([data])
    combine_args = [[v, k] + _get_extra_args(extra_args[k], combine_arg_keys)
                    for (k, v) in out_map.items()]
    return combine_args, list(final_args.values()) + extras
284685
def _do_merge(orig_files, out_file, config, region):
    
    if not utils.file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            _check_samples_nodups(orig_files)
            prep_files = run_multicore(p_bgzip_and_index, [[x, config] for x in orig_files], config)
            input_vcf_file = "%s-files.txt" % utils.splitext_plus(out_file)[0]
            with open(input_vcf_file, "w") as out_handle:
                for fname in prep_files:
                    out_handle.write(fname + "\n")
            bcftools = config_utils.get_program("bcftools", config)
            output_type = "z" if out_file.endswith(".gz") else "v"
            region_str = "-r {}".format(region) if region else ""
            cmd = "{bcftools} merge -O {output_type} {region_str} `cat {input_vcf_file}` > {tx_out_file}"
            do.run(cmd.format(**locals()), "Merge variants")
    if out_file.endswith(".gz"):
        bgzip_and_index(out_file, config)
    return out_file
909489
def concat_main(args):
    

    # Get files directly
    if args.input_filepaths is not None:
        files = args.input_filepaths

    # Or find them
    else:
        files = get_file_list(args.file_wildcard)
        
        # No files found
        if len(files) == 0:
            msg = "No files were found. args.file_wildcard: {}".format(args.file_wildcard)
            logger.error(msg)
            raise Exception(msg)

    # Only 1 file found
    if len(files) == 1:
        logger.warning("Only 1 file found. No concatenation needs to be done, exiting")
        return

    # More than 1 file found
    else:
        # Parse each file and append to a list
        gctoos = []
        for f in files:
            gctoos.append(parse.parse(f))

        # Create concatenated gctoo object
        if args.concat_direction == "horiz":
            out_gctoo = hstack(gctoos, args.remove_all_metadata_fields, args.error_report_output_file,
                               args.fields_to_remove, args.reset_ids)

        elif args.concat_direction == "vert":
            out_gctoo = vstack(gctoos, args.remove_all_metadata_fields, args.error_report_output_file,
                               args.fields_to_remove, args.reset_ids)

    # Write out_gctoo to file
    logger.info("Writing to output file args.out_name:  {}".format(args.out_name))

    if args.out_type == "gctx":
        write_gctx.write(out_gctoo, args.out_name)

    elif args.out_type == "gct":
        write_gct.write(out_gctoo, args.out_name,
                          filler_null=args.filler_null,
                          metadata_null=args.metadata_null,
                          data_null=args.data_null)
1338724
def merge!(*args)
      val = args.pop
      keys = args.flatten
      StorageFile.merge!(path, [*@parents, *keys], val)
    end
1315678
def _merge_files(self, input_files, output_file):
        """"""
        # we assume that all the input files have the same charset
        with open(output_file, mode='wb') as out:
            for input_file in input_files:
                out.write(open(input_file, mode='rb').read())
11993
def merge(args):
    
    p = OptionParser(merge.__doc__)
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    bedfiles = args
    fw = must_open(opts.outfile, "w")
    for bedfile in bedfiles:
        bed = Bed(bedfile)
        pf = op.basename(bedfile).split(".")[0]
        for b in bed:
            b.seqid = "_".join((pf, b.seqid))
            print(b, file=fw)
148325
def merge(args):
    
    p = OptionParser(merge.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 2:
        sys.exit(not p.print_help())

    binfiles = args[:-1]
    mergedbin = args[-1]
    if op.exists(mergedbin):
        logging.error("`{0}` file exists. Remove before proceed."\
                .format(mergedbin))
        return

    b = BinFile(binfiles[0])
    ar = b.mmarray
    fastasize, = ar.shape
    logging.debug("Initialize array of uint16 with size {0}".format(fastasize))

    merged_ar = np.zeros(fastasize, dtype=np.uint16)
    for binfile in binfiles:
        b = BinFile(binfile)
        merged_ar += b.array

    logging.debug("Resetting the count max to 255.")
    merged_ar[merged_ar > 255] = 255

    logging.debug("Compact array back to uint8 with size {0}".format(fastasize))
    merged_ar = np.array(merged_ar, dtype=np.uint8)
    merged_ar.tofile(mergedbin)
    logging.debug("Merged array written to `{0}`".format(mergedbin))
1332799
def _get_split_tasks(args, split_fn, file_key, outfile_i=-1):
    
    split_args = []
    combine_map = {}
    finished_map = collections.OrderedDict()
    extras = []
    for data in args:
        out_final, out_parts = split_fn(data)
        for parts in out_parts:
            split_args.append([utils.deepish_copy(data)] + list(parts))
        for part_file in [x[outfile_i] for x in out_parts]:
            combine_map[part_file] = out_final
        if len(out_parts) == 0:
            if out_final is not None:
                if out_final not in finished_map:
                    data[file_key] = out_final
                    finished_map[out_final] = [data]
                else:
                    extras.append([data])
            else:
                extras.append([data])
    return split_args, combine_map, list(finished_map.values()), extras
285787
def merge_required_files(dirnames, out_dir):
    
    # The list of files to merge
    fn_to_merge = ("steps_summary.tex", "excluded_markers.txt",
                   "excluded_samples.txt")

    # Merging the files
    for fn in fn_to_merge:
        o_fn = os.path.join(out_dir, fn)
        with open(o_fn, "w") as o_file:
            for dn in dirnames:
                i_fn = os.path.join(dn, fn)
                with open(i_fn, "r") as i_file:
                    o_file.write(i_file.read())

    # Merging the result summary file
    o_fn = os.path.join(out_dir, "results_summary.txt")
    with open(o_fn, "w") as o_file:
        for i, dn in enumerate(dirnames):
            i_fn = os.path.join(dn, "results_summary.txt")
            with open(i_fn, "r") as i_file:
                if i != 0:
                    # We skip the first 4 lines (file descriptions)
                    [i_file.readline() for i in range(4)]
                o_file.write(i_file.read())

    # Merging the graphic paths file
    graphic_paths = set()
    for dn in dirnames:
        fn = os.path.join(dn, "graphic_paths.txt")
        if os.path.isfile(fn):
            with open(fn, "r") as i_file:
                graphic_paths.update({
                    os.path.join(dn, path)
                    for path in i_file.read().splitlines()
                })
    if len(graphic_paths) > 0:
        with open(os.path.join(out_dir, "graphic_paths.txt"), "w") as o_file:
            for path in sorted(graphic_paths):
                print >>o_file, os.path.relpath(path, out_dir)
868611
def concatenating_merge(options, options2, *args)
      options = call_each_hash_value_with_params(options, *args) || {}
      options2 = call_each_hash_value_with_params(options2, *args) || {}


      options.symbolize_keys.merge(options2.symbolize_keys) do |key, v1, v2|
        if v1.is_a?(String) && v2.is_a?(String)
          "#{v1} #{v2}"
        else
          v2
        end
      end
    end
1344654
def merge_likelihood_headers(filenames, outfile):
    

    filenames = np.atleast_1d(filenames)

    ext='PIX_DATA'
    nside = fitsio.read_header(filenames[0],ext=ext)['LKDNSIDE']
    
    keys=['STELLAR','NINSIDE','NANNULUS']
    data_dict = odict(PIXEL=[])
    for k in keys:
        data_dict[k] = []

    for i,filename in enumerate(filenames):
        logger.debug('(%i/%i) %s'%(i+1, len(filenames), filename))
        header = fitsio.read_header(filename,ext=ext)
        data_dict['PIXEL'].append(header['LKDPIX'])
        for key in keys:
            data_dict[key].append(header[key])

        del header
        
    data_dict['PIXEL'] = np.array(data_dict['PIXEL'],dtype=int)
    for key in keys:
        data_dict[key] = np.array(data_dict[key],dtype='f4')

    #import pdb; pdb.set_trace()
    write_partial_map(outfile, data_dict, nside)
    return data_dict
507638
def _combine_files(tsv_files, work_dir, data):
    
    header = "\t".join(["caller", "sample", "chrom", "start", "end", "svtype",
                        "lof", "annotation", "split_read_support", "paired_support_PE", "paired_support_PR"])
    sample = dd.get_sample_name(data)
    out_file = os.path.join(work_dir, "%s-prioritize.tsv" % (sample))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            tmpdir = os.path.dirname(tx_out_file)
            input_files = " ".join(tsv_files)
            sort_cmd = bedutils.get_sort_cmd(tmpdir)
            cmd = "{{ echo '{header}'; cat {input_files} | {sort_cmd} -k3,3 -k4,4n; }} > {tx_out_file}"
            do.run(cmd.format(**locals()), "Combine prioritized from multiple callers")
    return out_file
474353
def mergebed(args):
    
    p = OptionParser(mergebed.__doc__)
    p.add_option("-w", "--weightsfile", default="weights.txt",
                 help="Write weights to file")
    p.set_outfile("out.bed")
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    maps = args
    outfile = opts.outfile
    fp = must_open(maps)
    b = Bed()
    mapnames = set()
    for row in fp:
        mapname = filename_to_mapname(fp.filename())
        mapnames.add(mapname)
        try:
            m = BedLine(row)
            m.accn = "{0}-{1}".format(mapname, m.accn)
            m.extra = ["{0}:{1}".format(m.seqid, m.start)]
            b.append(m)
        except (IndexError, ValueError):  # header or mal-formed line
            continue

    b.print_to_file(filename=outfile, sorted=True)
    logging.debug("A total of {0} markers written to `{1}`.".\
                        format(len(b), outfile))

    assert len(maps) == len(mapnames), "You have a collision in map names"
    write_weightsfile(mapnames, weightsfile=opts.weightsfile)
1340513
def merge_conf_file(self, result, conf_file_path):
        ""
        conf = parse_conf_file(conf_file_path)
        conf_file_name = os.path.splitext(os.path.basename(conf_file_path))[0]
        result_part = result

        if not conf_file_name in File.TOP_LEVEL_CONF_FILES \
            and (
                not "top_level" in self._options
                or not self._options["top_level"]):
            for key_part in conf_file_name.split('.'):
                if not key_part in result_part:
                    result_part[key_part] = {}
                result_part = result_part[key_part]

        return merge_conf(result_part, conf)
230452
def add_output_file(self, filename):
    
    if filename not in self.__output_files:
      self.__output_files.append(filename)
      if not isinstance(self.job(), CondorDAGManJob):
        if self.job().get_universe() == 'grid':
          self.add_output_macro(filename)
191195
def printArchive(fileName):
    
    archive = CombineArchive()
    if archive.initializeFromArchive(fileName) is None:
        print("Invalid Combine Archive")
        return None

    print('*'*80)
    print('Print archive:', fileName)
    print('*' * 80)
    printMetaDataFor(archive, ".")
    print("Num Entries: {0}".format(archive.getNumEntries()))

    for i in range(archive.getNumEntries()):
        entry = archive.getEntry(i)
        print(" {0}: location: {1} format: {2}".format(i, entry.getLocation(), entry.getFormat()))
        printMetaDataFor(archive, entry.getLocation())

        for j in range(entry.getNumCrossRefs()):
            print("  {0}: crossRef location {1}".format(j, entry.getCrossRef(j).getLocation()))

        # the entry could now be extracted via
        # archive.extractEntry(entry.getLocation(), <filename or folder>)

        # or used as string
        # content = archive.extractEntryToString(entry.getLocation());

    archive.cleanUp()
926685
def picard_merge(picard, in_files, out_file=None,
                 merge_seq_dicts=False):
    
    if out_file is None:
        out_file = "%smerge.bam" % os.path.commonprefix(in_files)
    if not file_exists(out_file):
        with tx_tmpdir(picard._config) as tmp_dir:
            with file_transaction(picard._config, out_file) as tx_out_file:
                opts = [("OUTPUT", tx_out_file),
                        ("SORT_ORDER", "coordinate"),
                        ("MERGE_SEQUENCE_DICTIONARIES",
                         "true" if merge_seq_dicts else "false"),
                        ("USE_THREADING", "true"),
                        ("TMP_DIR", tmp_dir)]
                for in_file in in_files:
                    opts.append(("INPUT", in_file))
                picard.run("MergeSamFiles", opts)
    return out_file