Merge all the blocks in the parity file into source file.
Source file will be converted into INodeRaidStorage format to include both
parity blocks and source blocks
978815
public void mergeInternal(INode parityINodes[], INode sourceINodes[],
      String parity, String source, RaidCodec codec, int[] checksums)
          throws IOException {
    waitForReady();
    long now = FSNamesystem.now();
    unprotectedMerge(parityINodes, sourceINodes, parity, source, codec,
        checksums, now);
    fsImage.getEditLog().logMerge(parity, source, codec.id, checksums, now);
  }
808739
def merge_files(sources, destination):
    

    with open(destination, 'w') as hout:
        for f in sources:
            if os.path.exists(f):
                with open(f) as hin:
                    shutil.copyfileobj(hin, hout)
            else:
                logger.warning('File is missing: {}'.format(f))
557122
def _consolidate_repo_sources(sources):
    '''
    
    '''
    if not isinstance(sources, sourceslist.SourcesList):
        raise TypeError(
            '\'{0}\' not a \'{1}\''.format(
                type(sources),
                sourceslist.SourcesList
            )
        )

    consolidated = {}
    delete_files = set()
    base_file = sourceslist.SourceEntry('').file

    repos = [s for s in sources.list if not s.invalid]

    for repo in repos:
        repo.uri = repo.uri.rstrip('/')
        # future lint: disable=blacklisted-function
        key = str((getattr(repo, 'architectures', []),
                   repo.disabled, repo.type, repo.uri, repo.dist))
        # future lint: enable=blacklisted-function
        if key in consolidated:
            combined = consolidated[key]
            combined_comps = set(repo.comps).union(set(combined.comps))
            consolidated[key].comps = list(combined_comps)
        else:
            consolidated[key] = sourceslist.SourceEntry(salt.utils.pkg.deb.strip_uri(repo.line))

        if repo.file != base_file:
            delete_files.add(repo.file)

    sources.list = list(consolidated.values())
    sources.save()
    for file_ in delete_files:
        try:
            os.remove(file_)
        except OSError:
            pass
    return sources
1196898
def combine_files(joinfile, sourcefiles, client):
    
    from mediasync.conf import msettings

    joinfile = joinfile.strip('/')

    if joinfile.endswith('.css'):
        dirname = msettings['CSS_PATH'].strip('/')
        separator = '\n'
    elif joinfile.endswith('.js'):
        dirname = msettings['JS_PATH'].strip('/')
        separator = ';\n'
    else:
        # By-pass this file since we only join CSS and JS.
        return None

    buffer = cStringIO.StringIO()

    for sourcefile in sourcefiles:
        sourcepath = os.path.join(client.media_root, dirname, sourcefile)
        if os.path.isfile(sourcepath):
            f = open(sourcepath)
            buffer.write(f.read())
            f.close()
            buffer.write(separator)

    filedata = buffer.getvalue()
    buffer.close()
    return (filedata, dirname)
342579
public void commitBlockSynchronization(Block lastblock,
                                  long newgenerationstamp, long newlength,
                                  boolean closeFile, boolean deleteblock, DatanodeID[] newtargets
  ) throws IOException {
    LOG.info("commitBlockSynchronization(lastblock=" + lastblock
      + ", newgenerationstamp=" + newgenerationstamp
      + ", newlength=" + newlength
      + ", newtargets=" + Arrays.asList(newtargets)
      + ", closeFile=" + closeFile
      + ", deleteBlock=" + deleteblock
      + ")");
    String src = null;
    writeLock();
    try {
      if (isInSafeMode()) {
        throw new SafeModeException("Cannot commitBlockSynchronization "
          + lastblock, safeMode);
      }      
      Block blockWithWildcardGenstamp = new Block(lastblock.getBlockId());
      final BlockInfo oldblockinfo = blocksMap.getStoredBlock(blockWithWildcardGenstamp);
      if (oldblockinfo == null) {
        throw new IOException("Block (=" + lastblock + ") not found");
      }
      if (!deleteblock
          && lastblock.getGenerationStamp() != oldblockinfo
              .getGenerationStamp()
          && oldblockinfo.getGenerationStamp() >= newgenerationstamp) {
        throw new IOException("Try to update block " + oldblockinfo
            + " to generation stamp " + newgenerationstamp);
      }
      INodeFile iFile = oldblockinfo.getINode();
      if (!iFile.isUnderConstruction()) {
        throw new IOException("Unexpected block (=" + lastblock
          + ") since the file (=" + iFile.getLocalName()
          + ") is not under construction");
      }
      INodeFileUnderConstruction pendingFile = (INodeFileUnderConstruction) iFile;
      // Fail the request if the block being synchronized is not the last block
      // of the file.
      pendingFile.checkLastBlockId(lastblock.getBlockId());

      // Remove old block from blocks map. This always have to be done
      // because the generation stamp of this block is changing.
      blocksMap.removeBlock(oldblockinfo);

      if (deleteblock) {
        pendingFile.removeBlock(lastblock);
      } else {
        // update last block, construct newblockinfo and add it to the blocks map
        lastblock.set(lastblock.getBlockId(), newlength, newgenerationstamp);
        final BlockInfo newblockinfo = 
            blocksMap.addINode(lastblock, pendingFile, 
                               pendingFile.getReplication());

        // find the DatanodeDescriptor objects
        // There should be no locations in the blocksMap till now because the
        // file is underConstruction
        DatanodeDescriptor[] descriptors = null;
        List<DatanodeDescriptor> descriptorsList =
          new ArrayList<DatanodeDescriptor>(newtargets.length);
        for(int i = 0; i < newtargets.length; i++) {
          // We don't use getDatanode here since that method can
          // throw. If we were to throw an exception during this commit
          // process, we'd fall out of sync since DNs have already finalized
          // the block with the new GS.
          DatanodeDescriptor node =
            datanodeMap.get(newtargets[i].getStorageID());
          if (node != null) {
            if (closeFile) {
              // If we aren't closing the file, we shouldn't add it to the
              // block list for the node, since the block is still under
              // construction there. (in getAdditionalBlock, for example
              // we don't add to the block map for the targets)
              node.addBlock(newblockinfo);
            }
            descriptorsList.add(node);
          } else {
          LOG.error("commitBlockSynchronization included a target DN " +
            newtargets[i] + " which is not known to NN. Ignoring.");
          }
        }
        if (!descriptorsList.isEmpty()) {
          descriptors = descriptorsList.toArray(new DatanodeDescriptor[0]);
        }
        // add locations into the INodeUnderConstruction
        pendingFile.setLastBlock(newblockinfo, descriptors);
      }

      // Get the file's full name; make sure that the file is valid
      src = pendingFile.getFullPathName();
      
      // If this commit does not want to close the file, persist
      // blocks only if append is supported and return
      if (!closeFile) {
        // persist the blocks if we support appends, 
        // or the block was deleted from the file and we need to persist it
        if (supportAppends || persistBlocks) {
          dir.persistBlocks(src, pendingFile);
        }
      } else {
        // remove lease, close file
        // this will persist the blocks
        finalizeINodeFileUnderConstruction(src, pendingFile);
      }
    } finally {// end of synchronized section
      writeUnlock();
    }

    if (closeFile || supportAppends) {
      getEditLog().logSync();
    }
    LOG.info("commitBlockSynchronization(newblock=" + lastblock
      + ", file=" + src + ") successful");
  }
881017
private void getMoreSplits(Configuration conf, Path[] paths, 
                             long maxSize, long minSizeNode, long minSizeRack,
                             List<InputSplit> splits)
    throws IOException {

    // all blocks for all the files in input set
    OneFileInfo[] files;
  
    // mapping from a rack name to the list of blocks it has
    HashMap<String, List<OneBlockInfo>> rackToBlocks = 
                              new HashMap<String, List<OneBlockInfo>>();

    // mapping from a block to the nodes on which it has replicas
    HashMap<OneBlockInfo, String[]> blockToNodes = 
                              new HashMap<OneBlockInfo, String[]>();

    // mapping from a node to the list of blocks that it contains
    HashMap<String, List<OneBlockInfo>> nodeToBlocks = 
                              new HashMap<String, List<OneBlockInfo>>();
    
    files = new OneFileInfo[paths.length];
    if (paths.length == 0) {
      return; 
    }

    // populate all the blocks for all files
    long totLength = 0;
    for (int i = 0; i < paths.length; i++) {
      files[i] = new OneFileInfo(paths[i], conf, 
                                 rackToBlocks, blockToNodes, nodeToBlocks, rackToNodes);
      totLength += files[i].getLength();
    }

    ArrayList<OneBlockInfo> validBlocks = new ArrayList<OneBlockInfo>();
    ArrayList<String> nodes = new ArrayList<String>();
    long curSplitSize = 0;

    // process all nodes and create splits that are local
    // to a node. 
    for (Iterator<Map.Entry<String, 
         List<OneBlockInfo>>> iter = nodeToBlocks.entrySet().iterator(); 
         iter.hasNext();) {

      Map.Entry<String, List<OneBlockInfo>> one = iter.next();
      nodes.add(one.getKey());
      List<OneBlockInfo> blocksInNode = one.getValue();

      // for each block, copy it into validBlocks. Delete it from 
      // blockToNodes so that the same block does not appear in 
      // two different splits.
      for (OneBlockInfo oneblock : blocksInNode) {
        if (blockToNodes.containsKey(oneblock)) {
          validBlocks.add(oneblock);
          blockToNodes.remove(oneblock);
          curSplitSize += oneblock.length;

          // if the accumulated split size exceeds the maximum, then 
          // create this split.
          if (maxSize != 0 && curSplitSize >= maxSize) {
            // create an input split and add it to the splits array
            addCreatedSplit(splits, nodes, validBlocks);
            curSplitSize = 0;
            validBlocks.clear();
          }
        }
      }
      // if there were any blocks left over and their combined size is
      // larger than minSplitNode, then combine them into one split.
      // Otherwise add them back to the unprocessed pool. It is likely 
      // that they will be combined with other blocks from the 
      // same rack later on.
      if (minSizeNode != 0 && curSplitSize >= minSizeNode) {
        // create an input split and add it to the splits array
        addCreatedSplit(splits, nodes, validBlocks);
      } else {
        for (OneBlockInfo oneblock : validBlocks) {
          blockToNodes.put(oneblock, oneblock.hosts);
        }
      }
      validBlocks.clear();
      nodes.clear();
      curSplitSize = 0;
    }

    // if blocks in a rack are below the specified minimum size, then keep them
    // in 'overflow'. After the processing of all racks is complete, these 
    // overflow blocks will be combined into splits.
    ArrayList<OneBlockInfo> overflowBlocks = new ArrayList<OneBlockInfo>();
    ArrayList<String> racks = new ArrayList<String>();

    // Process all racks over and over again until there is no more work to do.
    while (blockToNodes.size() > 0) {

      // Create one split for this rack before moving over to the next rack. 
      // Come back to this rack after creating a single split for each of the 
      // remaining racks.
      // Process one rack location at a time, Combine all possible blocks that
      // reside on this rack as one split. (constrained by minimum and maximum
      // split size).

      // iterate over all racks 
      for (Iterator<Map.Entry<String, List<OneBlockInfo>>> iter = 
           rackToBlocks.entrySet().iterator(); iter.hasNext();) {

        Map.Entry<String, List<OneBlockInfo>> one = iter.next();
        racks.add(one.getKey());
        List<OneBlockInfo> blocks = one.getValue();

        // for each block, copy it into validBlocks. Delete it from 
        // blockToNodes so that the same block does not appear in 
        // two different splits.
        boolean createdSplit = false;
        for (OneBlockInfo oneblock : blocks) {
          if (blockToNodes.containsKey(oneblock)) {
            validBlocks.add(oneblock);
            blockToNodes.remove(oneblock);
            curSplitSize += oneblock.length;
      
            // if the accumulated split size exceeds the maximum, then 
            // create this split.
            if (maxSize != 0 && curSplitSize >= maxSize) {
              // create an input split and add it to the splits array
              addCreatedSplit(splits, getHosts(racks), validBlocks);
              createdSplit = true;
              break;
            }
          }
        }

        // if we created a split, then just go to the next rack
        if (createdSplit) {
          curSplitSize = 0;
          validBlocks.clear();
          racks.clear();
          continue;
        }

        if (!validBlocks.isEmpty()) {
          if (minSizeRack != 0 && curSplitSize >= minSizeRack) {
            // if there is a minimum size specified, then create a single split
            // otherwise, store these blocks into overflow data structure
            addCreatedSplit(splits, getHosts(racks), validBlocks);
          } else {
            // There were a few blocks in this rack that 
        	// remained to be processed. Keep them in 'overflow' block list. 
        	// These will be combined later.
            overflowBlocks.addAll(validBlocks);
          }
        }
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }
    }

    assert blockToNodes.isEmpty();
    assert curSplitSize == 0;
    assert validBlocks.isEmpty();
    assert racks.isEmpty();

    // Process all overflow blocks
    for (OneBlockInfo oneblock : overflowBlocks) {
      validBlocks.add(oneblock);
      curSplitSize += oneblock.length;

      // This might cause an exiting rack location to be re-added,
      // but it should be ok.
      for (int i = 0; i < oneblock.racks.length; i++) {
        racks.add(oneblock.racks[i]);
      }

      // if the accumulated split size exceeds the maximum, then 
      // create this split.
      if (maxSize != 0 && curSplitSize >= maxSize) {
        // create an input split and add it to the splits array
        addCreatedSplit(splits, getHosts(racks), validBlocks);
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }
    }

    // Process any remaining blocks, if any.
    if (!validBlocks.isEmpty()) {
      addCreatedSplit(splits, getHosts(racks), validBlocks);
    }
  }
353632
public boolean encodeTmpParityFile(Configuration jobConf, StripeReader sReader,
      FileSystem parityFs, Path partialTmpParity, Path parityFile,
      short tmpRepl, long blockSize, long expectedPartialParityBlocks,
      long expectedPartialParityFileSize, Progressable reporter)
          throws IOException, InterruptedException {
    // Create a tmp file to which we will write first.
    String jobID = RaidNode.getJobID(jobConf);
    Path tmpDir = new Path(codec.tmpParityDirectory, jobID);
    if (!parityFs.mkdirs(tmpDir)) {
      throw new IOException("Could not create tmp dir " + tmpDir);
    }
    Path parityTmp = new Path(tmpDir, parityFile.getName() + rand.nextLong());
    FSDataOutputStream out = parityFs.create(
                               parityTmp,
                               true,
                               conf.getInt("io.file.buffer.size", 64 * 1024),
                               tmpRepl,
                               blockSize);
    try {
      CRC32[] crcOuts = null;
      if (checksumStore != null) {
        crcOuts = new CRC32[(int)expectedPartialParityBlocks];
      }
      encodeFileToStream(sReader, blockSize, out, crcOuts, reporter);
      out.close();
      out = null;
      LOG.info("Wrote temp parity file " + parityTmp);
      FileStatus tmpStat = parityFs.getFileStatus(parityTmp);
      if (tmpStat.getLen() != expectedPartialParityFileSize) {
        InjectionHandler.processEventIO(
            InjectionEvent.RAID_ENCODING_FAILURE_PARTIAL_PARITY_SIZE_MISMATCH);
        throw new IOException("Expected partial parity size " +
          expectedPartialParityFileSize + " does not match actual " +
          tmpStat.getLen() + " in path " + tmpStat.getPath());
      }
      InjectionHandler.processEventIO(
          InjectionEvent.RAID_ENCODING_FAILURE_PUT_CHECKSUM);
      if (checksumStore != null) {
        this.writeToChecksumStore((DistributedFileSystem)parityFs, crcOuts,
            parityTmp, expectedPartialParityFileSize, reporter);
      }
      if (!parityFs.rename(parityTmp, partialTmpParity)) {
        LOG.warn("Fail to rename file " + parityTmp + " to " + partialTmpParity);
        return false;
      }
      LOG.info("renamed " + parityTmp + " to " + partialTmpParity);
      return true;
    } finally {
      try {
        if (out != null) {
          out.close();
        }
      } finally {
        parityFs.delete(parityTmp, false);
      }
    }
  }
38229
private int addIndexedSplits(
			List<InputSplit> splits, int i, List<InputSplit> newSplits,
			Configuration cfg)
		throws IOException
	{
		final Path file = ((FileSplit)splits.get(i)).getPath();
		List<InputSplit> potentialSplits = new ArrayList<InputSplit>();

		final SplittingBAMIndex idx = new SplittingBAMIndex(
			file.getFileSystem(cfg).open(getIdxPath(file)));

		int splitsEnd = splits.size();
		for (int j = i; j < splitsEnd; ++j)
			if (!file.equals(((FileSplit)splits.get(j)).getPath()))
				splitsEnd = j;

		if (idx.size() == 1) { // no alignments, only the file size, so no splits to add
			return splitsEnd;
		}

		for (int j = i; j < splitsEnd; ++j) {
			final FileSplit fileSplit = (FileSplit)splits.get(j);

			final long start =         fileSplit.getStart();
			final long end   = start + fileSplit.getLength();

			final Long blockStart = idx.nextAlignment(start);

			// The last split needs to end where the last alignment ends, but the
			// index doesn't store that data (whoops); we only know where the last
			// alignment begins. Fortunately there's no need to change the index
			// format for this: we can just set the end to the maximal length of
			// the final BGZF block (0xffff), and then read until BAMRecordCodec
			// hits EOF.
			Long blockEnd;
			if (j == splitsEnd - 1) {
				blockEnd = idx.prevAlignment(end) | 0xffff;
			} else {
				blockEnd = idx.nextAlignment(end);
			}

			if (blockStart == null || blockEnd == null) {
				logger.warn("Index for {} was not good. Generating probabilistic splits.", file);
				return addProbabilisticSplits(splits, i, newSplits, cfg);
			}

			potentialSplits.add(new FileVirtualSplit(
						file, blockStart, blockEnd, fileSplit.getLocations()));
		}

		for (InputSplit s : potentialSplits) {
			newSplits.add(s);
		}
		return splitsEnd;
	}
407506
private static  void mergeBlock(FatCatParameters params, AFPChain afpChain,Atom[] ca1,Atom[] ca2)
	{

		int blockNum = afpChain.getBlockNum();
		double badRmsd = params.getBadRmsd();

		int[] block2Afp = afpChain.getBlock2Afp();
		int[] blockSize = afpChain.getBlockSize();

		double[] blockRmsd = afpChain.getBlockRmsd();

		int afpChainTwiNum = afpChain.getAfpChainTwiNum();

		//clustering the neighbor blocks if their transformations are similar
		int     i, j, b1, b2, minb1, minb2;
		double  minrmsd;
		int     merge = 0;
		int     blockNumOld = blockNum;
		double[][]  rmsdlist = null;
		if(blockNum > 1)        {
			rmsdlist = new double[blockNumOld][blockNumOld];
			for(b1 = 0; b1 < blockNum - 1; b1 ++)   {
				for(b2 = b1 + 1; b2 < blockNum; b2 ++)  {
					rmsdlist[b1][b2] = combineRmsd(b1, b2,afpChain,ca1,ca2);
				}
			}
		}
		minb1 = 0;
		while(blockNum > 1)     {
			minrmsd = 1000;
			for(i = 0; i < blockNum - 1; i ++)      {
				j = i + 1; //only consider neighbor blocks
				if(minrmsd > rmsdlist[i][j])    {
					minrmsd = rmsdlist[i][j];
					minb1 = i;
				}
			}
			minb2 = minb1 + 1; //merge those most similar blocks
			//maxrmsd = (blockRmsd[minb1] > blockRmsd[minb2])?blockRmsd[minb1]:blockRmsd[minb2];
			if(minrmsd < badRmsd)   {
				if(debug)
					System.out.println(String.format("merge block %d (rmsd %.3f) and %d (rmsd %.3f), total rmsd %.3f\n",
							minb1, blockRmsd[minb1], minb2, blockRmsd[minb2], minrmsd));
				blockSize[minb1] += blockSize[minb2];
				blockRmsd[minb1] = minrmsd;
				for(i = minb2; i < blockNum - 1; i ++)  {
					block2Afp[i] = block2Afp[i + 1];
					blockSize[i] = blockSize[i + 1];
					blockRmsd[i] = blockRmsd[i + 1];
				} //update block information
				afpChainTwiNum --;
				blockNum --;
				for(b1 = 0; b1 < blockNum - 1; b1 ++)   {
					for(b2 = b1 + 1; b2 < blockNum; b2 ++) {
						if(b1 == minb1 || b2 == minb1)  {
							rmsdlist[b1][b2] = combineRmsd(b1, b2, afpChain,ca1,ca2);
						}
						else if(b2 < minb1)     continue;
						else if(b1 < minb1)     {
							rmsdlist[b1][b2] = rmsdlist[b1][b2 + 1];
						}
						else    {
							rmsdlist[b1][b2] = rmsdlist[b1 + 1][b2 + 1];
						}
					}
				} //update the rmsdlist
				merge ++;
			} //merge two blocks
			else if(minrmsd >= 100) break;
			else    {
				rmsdlist[minb1][minb2] += 100;
			} //not merge, modify the rmsdlist so that this combination is not considered in next iteration
		}

		if(merge > 0)       {
			if(debug)
				System.out.println(String.format("Merge %d blocks, remaining %d blocks\n", merge, blockNum));
		}

		if (debug){
			System.err.println("AFPPostProcessor: mergeBlock end blocknum:" + blockNum);
		}
		afpChain.setBlock2Afp(block2Afp);
		afpChain.setBlockSize(blockSize);
		afpChain.setBlockNum(blockNum);
		afpChain.setBlockRmsd(blockRmsd);
		afpChain.setAfpChainTwiNum(afpChainTwiNum);
	}
802536
def merge(self, po_file, source_files):
        
        # Create a temporary file to write pot file
        pot_file = tempfile.NamedTemporaryFile(mode='wb', prefix='rookout_', delete=False)
        pot_filename = pot_file.name
        slog.info('Create POT file [%s].', pot_filename)
        xargs = [self._xgettext,
                "--package-name=main",
                "--package-version=0.1",
                "--default-domain=main",
                "--from-code=UTF-8",
                "-C", "-k_",
                "--output", pot_filename]
        txt = subprocess.check_output(xargs+source_files, 
                stderr=subprocess.STDOUT, 
                universal_newlines=True)
        if len(txt) > 0:
            raise(ChildProcessError(txt))
        slog.info('Start merge [%s] to [%s].', pot_filename, po_file)
        xargs = [self._msgmerge, "-U", po_file, pot_filename]
        txt = subprocess.check_output(xargs, universal_newlines=True)
        slog.info(txt)
        pot_file.close()
        os.remove(pot_filename)
157020
private int addIndexedSplits(
			List<InputSplit> splits, int i, List<InputSplit> newSplits,
			Configuration cfg)
		throws IOException
	{
		final Path file = ((FileSplit)splits.get(i)).getPath();

		final BGZFBlockIndex idx = new BGZFBlockIndex(
			file.getFileSystem(cfg).open(getIdxPath(file)));

		int splitsEnd = splits.size();
		for (int j = i; j < splitsEnd; ++j)
			if (!file.equals(((FileSplit)splits.get(j)).getPath()))
				splitsEnd = j;

		for (int j = i; j < splitsEnd; ++j) {
			final FileSplit fileSplit = (FileSplit)splits.get(j);

			final long start =         fileSplit.getStart();
			final long end   = start + fileSplit.getLength();

			final Long blockStart = idx.prevBlock(start);
			final Long blockEnd   = j == splitsEnd-1 ? idx.prevBlock(end)
			                                         : idx.nextBlock(end);

			if (blockStart == null)
				throw new RuntimeException(
					"Internal error or invalid index: no block start for " +start);

			if (blockEnd == null)
				throw new RuntimeException(
					"Internal error or invalid index: no block end for " +end);

			newSplits.add(new FileSplit(
				file, blockStart, blockEnd - blockStart,
				fileSplit.getLocations()));
		}
		return splitsEnd;
	}
373827
def tokenize_block(iterable, token_types):
    
    lines = FileWrapper(iterable)
    parse_buffer = ParseBuffer()
    line = lines.peek()
    while line is not None:
        for token_type in token_types:
            if token_type.start(line):
                result = token_type.read(lines)
                if result is not None:
                    parse_buffer.append((token_type, result))
                    break
        else:  # unmatched newlines
            next(lines)
            parse_buffer.loose = True
        line = lines.peek()
    return parse_buffer
882400
public boolean processFile(Path srcPath, ParityFilePair parityPair,
      Decoder decoder, Boolean fromStripeStore, Context context) 
          throws IOException,
      InterruptedException {
    LOG.info("Processing file " + srcPath);
    Progressable progress = context;
    if (progress == null) {
      progress = RaidUtils.NULL_PROGRESSABLE;
    }
    
    DistributedFileSystem srcFs = getDFS(srcPath);
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    long blockSize = srcStat.getBlockSize();
    long srcFileSize = srcStat.getLen();
    String uriPath = srcPath.toUri().getPath();

    int numBlocksReconstructed = 0;
    List<LocatedBlockWithMetaInfo> lostBlocks = lostBlocksInFile(srcFs,
        uriPath, srcStat);
    if (lostBlocks.size() == 0) {
      LOG.warn("Couldn't find any lost blocks in file " + srcPath + 
          ", ignoring...");
      return false;
    }
    List<Block> blocksLostChecksum = new ArrayList<Block>();
    List<Block> blocksLostStripe = new ArrayList<Block>();
    
    for (LocatedBlockWithMetaInfo lb: lostBlocks) {
      Block lostBlock = lb.getBlock();
      long lostBlockOffset = lb.getStartOffset();

      LOG.info("Found lost block " + lostBlock +
          ", offset " + lostBlockOffset);
      Long oldCRC = decoder.retrieveChecksum(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context);
      if (abortReconstruction(oldCRC, decoder)) {
        blocksLostChecksum.add(lostBlock);
        continue;
      }
      StripeInfo si = decoder.retrieveStripe(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context, false);
      if (si == null && decoder.stripeStore != null) {
        blocksLostStripe.add(lostBlock);
        continue;
      }

      final long blockContentsSize =
        Math.min(blockSize, srcFileSize - lostBlockOffset);
      File localBlockFile =
        File.createTempFile(lostBlock.getBlockName(), ".tmp");
      localBlockFile.deleteOnExit();

      try {
        CRC32 crc = null;
        
        if (fromStripeStore) {
          crc = decoder.recoverBlockToFileFromStripeInfo(srcFs, srcPath,
              lostBlock, localBlockFile, blockSize,
              lostBlockOffset, blockContentsSize, si, context);
        } else {
          crc = decoder.recoverBlockToFile(srcFs, srcStat,
                        parityPair.getFileSystem(),
                        parityPair.getPath(), blockSize,
                        lostBlockOffset, localBlockFile,
                        blockContentsSize, si, context);
        }
        LOG.info("Recovered crc: " + ((crc == null)?null: crc.getValue()) +
            " expected crc:" + oldCRC);
        if (crc != null && oldCRC != null &&
            crc.getValue() != oldCRC) {
          // checksum doesn't match, it's dangerous to send it
          IOException ioe = new IOException("Block " + lostBlock.toString() +
              " new checksum " + crc.getValue() +
              " doesn't match the old one " + oldCRC);
          LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0,
              decoder.codec, srcPath, lostBlockOffset,
              LOGTYPES.OFFLINE_RECONSTRUCTION_CHECKSUM_VERIFICATION,
              srcFs, ioe, context);
          throw ioe;
        }
        // Now that we have recovered the file block locally, send it.
        computeMetadataAndSendReconstructedBlock(localBlockFile,
            lostBlock, blockContentsSize,
            lb.getLocations(),
            lb.getDataProtocolVersion(), 
            lb.getNamespaceID(), 
            progress);
        
        numBlocksReconstructed++;

      } finally {
        localBlockFile.delete();
      }
      progress.progress();
    }
    
    LOG.info("Reconstructed " + numBlocksReconstructed + " blocks in " + srcPath);
    checkLostBlocks(blocksLostChecksum, blocksLostStripe, srcPath, decoder.codec);
    return true;
  }
917073
private void mergeFiles(List<File> inputFiles, OutputStream outputStream)
      throws IOException {
    long startTime = System.nanoTime();

    //Merge files
    for (File f : inputFiles) {
      if (f.exists()) {
        FileInputStream fileInputStream = new FileInputStream(f);
        BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream);
        try {
          LOGGER.log(Level.INFO, "Merging {0} size={1}", new Object[]{f.getName(), f.length()});

          byte[] buffer = new byte[8192];
          int length;
          while ((length = bufferedInputStream.read(buffer)) > 0) {
            outputStream.write(buffer, 0, length);
          }
        } finally {
          bufferedInputStream.close();
          fileInputStream.close();
        }
      } else {
        LOGGER.log(Level.INFO, "Skip merging file {0} because it doesn't exist", f.getName());
      }
    }

    LOGGER.log(Level.INFO, "Time to merge {0} s", ((System.nanoTime() - startTime) / 1000000000.0));
  }
69624
def merge(self, workdir, ddb_files, out_ddb, description, delete_source_ddbs=True):
        """"""
        # We work with absolute paths.
        ddb_files = [os.path.abspath(s) for s in list_strings(ddb_files)]
        if not os.path.isabs(out_ddb):
            out_ddb = os.path.join(os.path.abspath(workdir), os.path.basename(out_ddb))

        if self.verbose:
            print("Will merge %d files into output DDB %s" % (len(ddb_files), out_ddb))
            for i, f in enumerate(ddb_files):
                print(" [%d] %s" % (i, f))

        # Handle the case of a single file since mrgddb uses 1 to denote GS files!
        if len(ddb_files) == 1:
            with open(ddb_files[0], "r") as inh, open(out_ddb, "w") as out:
                for line in inh:
                    out.write(line)
            return out_ddb

        self.stdin_fname, self.stdout_fname, self.stderr_fname = \
            map(os.path.join, 3 * [os.path.abspath(workdir)], ["mrgddb.stdin", "mrgddb.stdout", "mrgddb.stderr"])

        inp = StringIO()
        inp.write(out_ddb + "\n")              # Name of the output file.
        inp.write(str(description) + "\n")     # Description.
        inp.write(str(len(ddb_files)) + "\n")  # Number of input DDBs.

        # Names of the DDB files.
        for fname in ddb_files:
            inp.write(fname + "\n")

        self.stdin_data = [s for s in inp.getvalue()]

        with open(self.stdin_fname, "wt") as fh:
            fh.writelines(self.stdin_data)
            # Force OS to write data to disk.
            fh.flush()
            os.fsync(fh.fileno())

        retcode = self.execute(workdir, exec_args=['--nostrict'])
        if retcode == 0 and delete_source_ddbs:
            # Remove ddb files.
            for f in ddb_files:
                try:
                    os.remove(f)
                except IOError:
                    pass

        return out_ddb
870045
void mergeContiguousRegions( Collection<AbstractBlock> blocks ) {

		AbstractIgnorableInstruction initialInstr = null;
		List<AbstractBlock> toRemove = new ArrayList<> ();
		StringBuilder sb = new StringBuilder();

		// We only merge comments and blank regions to reduce their number
		for( AbstractBlock block : blocks ) {
			if( initialInstr == null ) {

				if( block.getInstructionType() == AbstractBlock.COMMENT
						|| block.getInstructionType() == AbstractBlock.BLANK ) {

					AbstractIgnorableInstruction currentInstr = (AbstractIgnorableInstruction) block;
					initialInstr = currentInstr;
					sb = new StringBuilder( currentInstr.getContent());
				}

			} else if( initialInstr.getInstructionType() == block.getInstructionType()) {
				toRemove.add( block );
				sb.append( System.getProperty( "line.separator" ));
				sb.append(((AbstractIgnorableInstruction) block).getContent());

			} else {
				initialInstr.setContent( sb.toString());
				initialInstr = null;
			}
		}

		// Remove the blocks that have been merged
		blocks.removeAll( toRemove );

		// In a second time, we can reduce facets and components too
		for( AbstractBlock block : blocks ) {
			if( block.getInstructionType() == AbstractBlock.COMPONENT
					|| block.getInstructionType() == AbstractBlock.FACET )
				mergeContiguousRegions(((AbstractBlockHolder) block).getInnerBlocks());
		}
	}
861446
def merge(args):
    
    from jcvi.apps.grid import MakeManager

    p = OptionParser(merge.__doc__)
    p.set_sep(sep="_", help="Separator to group per prefix")
    opts, args = p.parse_args(args)

    if len(args) < 2:
        sys.exit(not p.print_help())

    merged_bams = args[0]
    bamdirs = args[1:]

    mkdir(merged_bams)
    bams = []
    for x in bamdirs:
        bams += glob(op.join(x, "*.bam"))
    bams = [x for x in bams if "nsorted" not in x]

    logging.debug("Found a total of {0} BAM files.".format(len(bams)))

    sep = opts.sep
    key = lambda x: op.basename(x).split(sep)[0]
    bams.sort(key=key)
    mm = MakeManager()
    for prefix, files in groupby(bams, key=key):
        files = sorted(list(files))
        nfiles = len(files)
        source = " ".join(files)
        target = op.join(merged_bams, op.basename(files[0]))
        if nfiles == 1:
            source = get_abs_path(source)
            cmd = "ln -s {0} {1}".format(source, target)
            mm.add("", target, cmd)
        else:
            cmd = "samtools merge -@ 8 {0} {1}".format(target, source)
            mm.add(files, target, cmd, remove=True)
    mm.write()