Merge(parity, source, ...) log
It's used for converting old raided files into new format
by merging parity file and source file together into one file
198624
public void logMerge(String parity, String source, String codecId, 
      int[] checksums, long timestamp) {
    MergeOp op = MergeOp.getInstance();
    op.set(parity, source, codecId, checksums, timestamp);
    logEdit(op);
  }
750524
public function mergeLog(Log $log)
	{
		/** @var Release $release */
		foreach ($log as $release)
		{
			$name = $release->getName();
			if ($this->hasRelease($name))
			{
				// if it does exist then merge the changes
				$this->mergeRelease($log, $name);
			}
			else
			{
				// If the release does not exist add it
				$this->addRelease($release);
			}
		}
	}
778581
def combine_chunks(storage, args, prefix=None):
    '''
    
    '''
    uuid = args['uuid']
    # Normalize filename including extension
    target = utils.normalize(args['filename'])
    if prefix:
        target = os.path.join(prefix, target)
    with storage.open(target, 'wb') as out:
        for i in xrange(args['totalparts']):
            partname = chunk_filename(uuid, i)
            out.write(chunks.read(partname))
            chunks.delete(partname)
    chunks.delete(chunk_filename(uuid, META))
    return target
292725
def merge_new_content(new_content, path, comment, kind)
      from_directory do
        log.debug "merging new content to #{path} of kind #{kind}"
        already_existing = File.exist?(path)

        generated_comment = "tetra: generated-#{kind}"
        whole_comment = [comment, generated_comment].join("\n\n")

        if already_existing
          unless @git.latest_id(generated_comment)
            log.debug "committing new file"
            @git.commit_file(path, whole_comment)
          end
          log.debug "moving #{path} to #{path}.tetra_user_edited"
          File.rename(path, "#{path}.tetra_user_edited")
        end

        previous_id = @git.latest_id(generated_comment)

        File.open(path, "w") { |io| io.write(new_content) }
        log.debug "committing new content: #{comment}"
        @git.commit_file(path, whole_comment)

        if already_existing
          # 3-way merge
          conflict_count = @git.merge_with_id(path, "#{path}.tetra_user_edited", previous_id)
          File.delete("#{path}.tetra_user_edited")

          @git.commit_file(path, "User changes merged back") if conflict_count == 0

          return conflict_count
        end
        return 0
      end
    end
939442
def merge_scans(workdir, fileroot, scans, snrmin=0, snrmax=999):
    


    pkllist = [ff for ff in
               [os.path.join(workdir, 'cands_{0}_sc{1}.pkl'.format(fileroot, scan))
                for scan in scans] if os.path.exists(ff)]
    merge_cands(pkllist, outroot=fileroot, snrmin=snrmin, snrmax=snrmax)

    pkllist = [ff for ff in
               [os.path.join(workdir, 'noise_{0}_sc{1}.pkl'.format(fileroot, scan))
                for scan in scans] if os.path.exists(ff)]
    merge_noises(pkllist, fileroot)
240311
private void merge(Versioning versioning, ArtifactRepository from, List<VersionInfo> found) {
        if ( StringUtils.isNotEmpty( versioning.getRelease() ) )
            found.add(new VersionInfo(versioning.getRelease(),from));

        if ( StringUtils.isNotEmpty( versioning.getLatest() ) )
            found.add(new VersionInfo(versioning.getLatest(),from));

        for ( String v : versioning.getVersions() )
            found.add(new VersionInfo(v,from));
    }

    VersionInfo findLatest(List<VersionInfo> infos) {
        Collections.sort(infos);
        if (infos.isEmpty())    return null;
        return infos.get(infos.size()-1);
    }

    private static class VersionInfo implements Comparable<VersionInfo> {
        final VersionNumber version;
        final ArtifactRepository repository;

        public VersionInfo(String version, ArtifactRepository repository) {
            this.version = new VersionNumber(version);
            this.repository = repository;
        }

        public int compareTo(VersionInfo that) {
            return this.version.compareTo(that.version);
        }
    }
    
    private Versioning readVersions( RepositorySystemSession session, RequestTrace trace, Metadata metadata,
                                     ArtifactRepository repository, VersionResult result )
    {
        Versioning versioning = null;

        FileInputStream fis = null;
        try
        {
            if ( metadata != null )
            {
                SyncContext syncContext = syncContextFactory.newInstance( session, true );

                try
                {
                    syncContext.acquire( null, Collections.singleton( metadata ) );

                    if ( metadata.getFile() != null && metadata.getFile().exists() )
                    {
                        fis = new FileInputStream( metadata.getFile() );
                        org.apache.maven.artifact.repository.metadata.Metadata m =
                            new MetadataXpp3Reader().read( fis, false );
                        versioning = m.getVersioning();

                        /*
                         * NOTE: Users occasionally misuse the id "local" for remote repos which screws up the metadata
                         * of the local repository. This is especially troublesome during snapshot resolution so we try
                         * to handle that gracefully.
                         */
                        if ( versioning != null && repository instanceof LocalRepository)
                        {
                            if ( versioning.getSnapshot() != null && versioning.getSnapshot().getBuildNumber() > 0 )
                            {
                                Versioning repaired = new Versioning();
                                repaired.setLastUpdated( versioning.getLastUpdated() );
                                Snapshot snapshot = new Snapshot();
                                snapshot.setLocalCopy( true );
                                repaired.setSnapshot( snapshot );
                                versioning = repaired;

                                throw new IOException( "Snapshot information corrupted with remote repository data"
                                    + ", please verify that no remote repository uses the id '" + repository.getId()
                                    + "'" );
                            }
                        }
                    }
                }
                finally
                {
                    syncContext.release();
                }
            }
        }
        catch ( Exception e )
        {
            invalidMetadata( session, trace, metadata, repository, e );
            result.addException( e );
        }
        finally
        {
            IOUtil.close(fis);
        }

        return ( versioning != null ) ? versioning : new Versioning();
    }

    private void invalidMetadata( RepositorySystemSession session, RequestTrace trace, Metadata metadata,
                                  ArtifactRepository repository, Exception exception )
    {
        RepositoryListener listener = session.getRepositoryListener();
        if ( listener != null )
        {
            DefaultRepositoryEvent event = new DefaultRepositoryEvent( EventType.METADATA_INVALID, session, trace );
            event.setMetadata( metadata );
            event.setException( exception );
            event.setRepository( repository );
            listener.metadataInvalid( event );
        }
    }

    private static final String MAVEN_METADATA_XML = "maven-metadata.xml";

    private static final String LATEST = "LATEST";

    private static final String SNAPSHOT = "SNAPSHOT";
}
855070
private static BlockLocation[] getParityBlocks(final Path filePath,
                                          final long blockSize,
                                          final long numStripes,
                                          final RaidInfo raidInfo) 
    throws IOException {
    FileSystem parityFS = raidInfo.parityPair.getFileSystem();
    
    // get parity file metadata
    FileStatus parityFileStatus = raidInfo.parityPair.getFileStatus(); 
    long parityFileLength = parityFileStatus.getLen();

    if (parityFileLength != numStripes * raidInfo.parityBlocksPerStripe *
        blockSize) {
      throw new IOException("expected parity file of length" + 
                            (numStripes * raidInfo.parityBlocksPerStripe *
                             blockSize) +
                            " but got parity file of length " + 
                            parityFileLength);
    }

    BlockLocation[] parityBlocks = 
      parityFS.getFileBlockLocations(parityFileStatus, 0L, parityFileLength);
    
    if (parityFS instanceof DistributedFileSystem ||
        parityFS instanceof DistributedRaidFileSystem) {
      long parityBlockSize = parityFileStatus.getBlockSize();
      if (parityBlockSize != blockSize) {
        throw new IOException("file block size is " + blockSize + 
                              " but parity file block size is " + 
                              parityBlockSize);
      }
    } else if (parityFS instanceof HarFileSystem) {
      LOG.debug("HAR FS found");
    } else {
      LOG.warn("parity file system is not of a supported type");
    }
    
    return parityBlocks;
  }
917073
private void mergeFiles(List<File> inputFiles, OutputStream outputStream)
      throws IOException {
    long startTime = System.nanoTime();

    //Merge files
    for (File f : inputFiles) {
      if (f.exists()) {
        FileInputStream fileInputStream = new FileInputStream(f);
        BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream);
        try {
          LOGGER.log(Level.INFO, "Merging {0} size={1}", new Object[]{f.getName(), f.length()});

          byte[] buffer = new byte[8192];
          int length;
          while ((length = bufferedInputStream.read(buffer)) > 0) {
            outputStream.write(buffer, 0, length);
          }
        } finally {
          bufferedInputStream.close();
          fileInputStream.close();
        }
      } else {
        LOGGER.log(Level.INFO, "Skip merging file {0} because it doesn't exist", f.getName());
      }
    }

    LOGGER.log(Level.INFO, "Time to merge {0} s", ((System.nanoTime() - startTime) / 1000000000.0));
  }
360870
public void merge(Path parity, Path source, String codecId, 
      int[] checksums) throws IOException {
    dfs.merge(getPathName(parity), getPathName(source), codecId, checksums);
  }
1239141
def merge_pairs(self):
        
        logging.info('Merging paired reads')
        for sample in self.metadata:
            # Can only merge paired-end
            if len(sample.general.fastqfiles) == 2:
                # Set the name of the merged, and unmerged files
                sample.general.mergedreads = \
                    os.path.join(sample.general.outputdirectory, '{}_paired.fastq.gz'.format(sample.name))
                sample.general.unmergedforward = \
                    os.path.join(sample.general.outputdirectory, '{}_unpaired_R1.fastq.gz'.format(sample.name))
                sample.general.unmergedreverse = \
                    os.path.join(sample.general.outputdirectory, '{}_unpaired_R2.fastq.gz'.format(sample.name))
                try:
                    # Run the merging command - forward_in=sample.general.normalisedreads[0],
                    out, err, cmd = bbtools.bbmerge(forward_in=sorted(sample.general.trimmedcorrectedfastqfiles)[0],
                                                    merged_reads=sample.general.mergedreads,
                                                    returncmd=True,
                                                    outu1=sample.general.unmergedforward,
                                                    outu2=sample.general.unmergedreverse,
                                                    threads=self.cpus)
                    sample[self.analysistype].bbmergecmd = cmd
                    write_to_logfile(out, err, self.logfile, sample.general.logout, sample.general.logerr, None, None)
                except CalledProcessError:
                    delattr(sample.general, 'mergedreads')
                    delattr(sample.general, 'unmergedforward')
                    delattr(sample.general, 'unmergedreverse')
                except IndexError:
                    delattr(sample.general, 'mergedreads')
                    delattr(sample.general, 'unmergedforward')
                    delattr(sample.general, 'unmergedreverse')
            else:
                sample.general.mergedreads = sorted(sample.general.trimmedcorrectedfastqfiles)[0]
185326
@Override
    protected void parse() {
        maximumInstances = getInt(PARAM_MAXIMUM_INSTANCES, DEFAULT_MAXIMUM_INSTANCES);
        mergeRelatedIssues = getBoolean(PARAM_MERGE_RELATED_ISSUES, true);
        overridesFilename = getString(PARAM_OVERRIDES_FILENAME, "");
    }
861446
def merge(args):
    
    from jcvi.apps.grid import MakeManager

    p = OptionParser(merge.__doc__)
    p.set_sep(sep="_", help="Separator to group per prefix")
    opts, args = p.parse_args(args)

    if len(args) < 2:
        sys.exit(not p.print_help())

    merged_bams = args[0]
    bamdirs = args[1:]

    mkdir(merged_bams)
    bams = []
    for x in bamdirs:
        bams += glob(op.join(x, "*.bam"))
    bams = [x for x in bams if "nsorted" not in x]

    logging.debug("Found a total of {0} BAM files.".format(len(bams)))

    sep = opts.sep
    key = lambda x: op.basename(x).split(sep)[0]
    bams.sort(key=key)
    mm = MakeManager()
    for prefix, files in groupby(bams, key=key):
        files = sorted(list(files))
        nfiles = len(files)
        source = " ".join(files)
        target = op.join(merged_bams, op.basename(files[0]))
        if nfiles == 1:
            source = get_abs_path(source)
            cmd = "ln -s {0} {1}".format(source, target)
            mm.add("", target, cmd)
        else:
            cmd = "samtools merge -@ 8 {0} {1}".format(target, source)
            mm.add(files, target, cmd, remove=True)
    mm.write()
284685
def _do_merge(orig_files, out_file, config, region):
    
    if not utils.file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            _check_samples_nodups(orig_files)
            prep_files = run_multicore(p_bgzip_and_index, [[x, config] for x in orig_files], config)
            input_vcf_file = "%s-files.txt" % utils.splitext_plus(out_file)[0]
            with open(input_vcf_file, "w") as out_handle:
                for fname in prep_files:
                    out_handle.write(fname + "\n")
            bcftools = config_utils.get_program("bcftools", config)
            output_type = "z" if out_file.endswith(".gz") else "v"
            region_str = "-r {}".format(region) if region else ""
            cmd = "{bcftools} merge -O {output_type} {region_str} `cat {input_vcf_file}` > {tx_out_file}"
            do.run(cmd.format(**locals()), "Merge variants")
    if out_file.endswith(".gz"):
        bgzip_and_index(out_file, config)
    return out_file
1345940
function readAndMergeFiles(files, cb) {
    assert.func(cb);
    if ((typeof (files)) === 'string') {
        files = [ files ];
    }
    assert.arrayOfString(files, 'files');
    var objs = [];
    function end() {
        cb(null, merge.apply(this, objs));
    }
    var i = 0;
    function next() {
        var file = files[i];
        if (!file) {
            end();
            return;
        }

        fs.stat(file, function (statErr, stat) {
            if (statErr && statErr.code === 'ENOENT' && i != 0) {
                ++i;
                next();
                return;
            }
            if (statErr) {
                cb(statErr);
                return;
            }

            readJsonFile(file, function (readErr, s) {
                if (readErr) {
                    cb(readErr);
                    return;
                }
                objs.push(s);
                ++i;
                next();
            });
        });
    }
    next();
}
1007265
private void transferLogFilesToS3() {
        if (_fileTransfersEnabled) {
            // Find all closed log files in the staging directory and move them to S3
            for (final File logFile : _stagingDir.listFiles((dir, name) -> name.startsWith(_logFilePrefix) && name.endsWith(COMPRESSED_FILE_SUFFIX))) {
                // Extract the date portion of the file name and is it to partition the file in S3
                String auditDate = logFile.getName().substring(_logFilePrefix.length() + 1, _logFilePrefix.length() + 9);
                String dest = String.format("%s/date=%s/%s", _s3AuditRoot, auditDate, logFile.getName());

                _fileTransferService.submit(() -> {
                    // Since file transfers are done in a single thread, there shouldn't be any concurrency issues,
                    // but verify the same file wasn't submitted previously and is already transferred.
                    if (logFile.exists()) {
                        try {
                            PutObjectResult result = _s3.putObject(_s3Bucket, dest, logFile);
                            _log.debug("Audit log copied: {}, ETag={}", logFile, result.getETag());

                            if (!logFile.delete()) {
                                _log.warn("Failed to delete file after copying to s3: {}", logFile);
                            }
                        } catch (Exception e) {

                            // Log the error, try again on the next iteration
                            _rateLimitedLog.error(e, "Failed to copy log file {}", logFile);
                        }
                    }
                });
            }
        }

    }
88068
def delayed_bam_merge(data):
    
    if data.get("combine"):
        assert len(data["combine"].keys()) == 1
        file_key = list(data["combine"].keys())[0]
        extras = []
        for x in data["combine"][file_key].get("extras", []):
            if isinstance(x, (list, tuple)):
                extras.extend(x)
            else:
                extras.append(x)
        if file_key in data:
            extras.append(data[file_key])
        in_files = sorted(list(set(extras)))
        out_file = tz.get_in(["combine", file_key, "out"], data, _merge_out_from_infiles(in_files))
        sup_exts = data.get(file_key + "_plus", {}).keys()
        for ext in list(sup_exts) + [""]:
            merged_file = None
            if os.path.exists(utils.append_stem(out_file, "-" + ext)):
                cur_out_file, cur_in_files = out_file, []
            if ext:
                cur_in_files = list(filter(os.path.exists, (utils.append_stem(f, "-" + ext) for f in in_files)))
                cur_out_file = utils.append_stem(out_file, "-" + ext) if len(cur_in_files) > 0 else None
            else:
                cur_in_files, cur_out_file = in_files, out_file
            if cur_out_file:
                config = copy.deepcopy(data["config"])
                if len(cur_in_files) > 0:
                    merged_file = merge_bam_files(cur_in_files, os.path.dirname(cur_out_file), data,
                                                  out_file=cur_out_file)
                else:
                    assert os.path.exists(cur_out_file)
                    merged_file = cur_out_file
            if merged_file:
                if ext:
                    data[file_key + "_plus"][ext] = merged_file
                else:
                    data[file_key] = merged_file
        data.pop("region", None)
        data.pop("combine", None)
    return [[data]]
1208692
function merge(input, output) {
  const text = fs.readFileSync(input, 'utf8');
  const loaded = yaml.safeLoad(text.toString(), { schema: yamlfiles.YAML_FILES_SCHEMA });

  // write to output
  fs.writeFileSync(output, yaml.safeDump(loaded));
  return output;
}