Returns the corrupt blocks in a file.
189628
public static List<LocatedBlock> corruptBlocksInFile(
    DistributedFileSystem dfs, String path, long offset, long length)
  throws IOException {
    List<LocatedBlock> corrupt = new LinkedList<LocatedBlock>();
    LocatedBlocks locatedBlocks =
      getBlockLocations(dfs, path, offset, length);
    for (LocatedBlock b: locatedBlocks.getLocatedBlocks()) {
      if (b.isCorrupt() ||
         (b.getLocations().length == 0 && b.getBlockSize() > 0)) {
        corrupt.add(b);
      }
    }
    return corrupt;
  }
587296
@Override @Deprecated
  public FileStatus[] getCorruptFiles() throws IOException {
    CorruptFileBlocks corruptFileBlocks = listCorruptFileBlocks("/", null);
    Set<String> filePaths = new HashSet<String>();
    for (String file : corruptFileBlocks.getFiles()) {
      filePaths.add(file);
    }

    List<FileStatus> fileStatuses = new ArrayList<FileStatus>(filePaths.size());
    for (String f: filePaths) {
      FileStatus fs = getFileInfo(f);
      if (fs != null)
        LOG.info("found fs for " + f);
      else
        LOG.info("found no fs for " + f);
      fileStatuses.add(fs);
    }
    return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
  }
466710
static long getGenerationStampFromInlineChecksumFile(String blockName)
      throws IOException {
    String[] vals = StringUtils.split(blockName, '_');
    if (vals.length != 6) {
      // blk, blkid, genstamp, version, checksumtype, byte per checksum
      throw new IOException("unidentified block name format: " + blockName);
    }
    return Long.parseLong(vals[2]);
  }
91922
def renumber_block_keys(blocks):
    

    # There is an implicit block switch to the 0th block at the start of the
    # file
    byte_switch_keys = [0]
    block_keys = list(blocks.keys())

    # Scan the blocks, recording every block switch statement
    for block in list(blocks.values()):
        i = 0
        while i < len(block.data) - 1:
            current_byte = block.data[i]
            next_byte = block.data[i + 1]

            if current_byte == RLE_BYTE:
                if next_byte == RLE_BYTE:
                    i += 2
                else:
                    i += 3
            elif current_byte == SPECIAL_BYTE:
                if next_byte in SPECIAL_DEFAULTS:
                    i += 3
                elif next_byte == SPECIAL_BYTE:
                    i += 2
                else:
                    if next_byte != EOF_BYTE:
                        byte_switch_keys.append(next_byte)

                    break

            else:
                i += 1

    byte_switch_keys.sort()
    block_keys.sort()

    assert len(byte_switch_keys) == len(block_keys), (
        "Number of blocks that are target of block switches (%d) "
        % (len(byte_switch_keys)) +
        "does not equal number of blocks in the song (%d)"
        % (len(block_keys)) +
        "; possible corruption")

    if byte_switch_keys == block_keys:
        # No remapping necessary
        return blocks

    new_block_map = {}

    for block_key, byte_switch_key in zip(
            block_keys, byte_switch_keys):

        new_block_map[byte_switch_key] = blocks[block_key]

    return new_block_map
823934
protected function readBlocks()
	{
		$open	= false;
		$file	= new FS_File_Reader( $this->fileName );
		$lines	= $file->readArray();
		foreach( $lines as $line )
		{
			$line	= trim( $line );
			if( $line )
			{
				if( preg_match( $this->patternSection, $line ) )
				{
					$section 	= preg_replace( $this->patternSection, "\\1", $line );
					if( !isset( $this->blocks[$section] ) )
						$this->blocks[$section]	= array();
					$open = true;
				}
				else if( $open )
				{
					$this->blocks[$section][]	= $line;
				}
			}
		}
		foreach( $this->blocks as $section => $block )
			$this->blocks[$section]	= implode( "\n", $block );
	}
67360
def _content_blocks(self, r):
    """"""
    return (self._block_rows - self._left_zero_blocks(r)
            - self._right_zero_blocks(r))
605427
def corrupt_bytes(s, p=0.01, n=None):
    """"""
    s = bytes(s)
    s = array.array("B",s)
    l = len(s)
    if n is None:
        n = max(1,int(l*p))
    for i in random.sample(range(l), n):
        s[i] = (s[i]+random.randint(1,255))%256
    return s.tobytes()
240636
void checkLostBlocks(List<Block> blocksLostChecksum,
      List<Block> blocksLostStripe, Path p, Codec codec)
      throws IOException {
    StringBuilder message = new StringBuilder();
    if (blocksLostChecksum.size() > 0) {
      message.append("Lost " + blocksLostChecksum.size() +
          " checksums in blocks:");
      for (Block blk : blocksLostChecksum) {
        message.append(" ");
        message.append(blk.toString());
      }
    }
    if (blocksLostStripe.size() > 0) {
      message.append("Lost " + blocksLostStripe.size() +
          " stripes in blocks:");
      for (Block blk : blocksLostStripe) {
        message.append(" ");
        message.append(blk.toString());
      }
    }
    if (message.length() == 0)
      return;
    message.append(" in file " + p);
    throw new IOException(message.toString());
  }
262059
def getblock(self, blockhash, decode=False):
        """"""

        if not decode:
            decode = "false"

            return self.req("getblock", [blockhash, decode])

        else:
            return self.req("getblock", [blockhash])
894943
def blockshapes(self):
        """"""
        if self._blockshapes is None:
            if self._filename:
                self._populate_from_rasterio_object(read_image=False)
            else:
                # if no file is attached to the raster set the shape of each band to be the data array size
                self._blockshapes = [(self.height, self.width) for z in range(self.num_bands)]
        return self._blockshapes
244382
def get_checksum(file):
    
    with open(file, 'rb') as FH:
        contents = FH.read()
    return hashlib.sha256(contents).hexdigest()
1027342
private static void checkParityBlocks(final Path filePath,
                                 final Map<Integer, Integer>
                                 corruptBlocksPerStripe,
                                 final long blockSize,
                                 final long startStripeIdx,
                                 final long endStripeIdx,
                                 final long numStripes,
                                 final RaidInfo raidInfo)
    throws IOException {

    // get the blocks of the parity file
    // because of har, multiple blocks may be returned as one container block
    BlockLocation[] containerBlocks = getParityBlocks(filePath, blockSize,
        numStripes, raidInfo);

    long parityStripeLength = blockSize *
      ((long) raidInfo.parityBlocksPerStripe);

    long parityBlocksFound = 0L;

    for (BlockLocation cb: containerBlocks) {
      if (cb.getLength() % blockSize != 0) {
        throw new IOException("container block size is not " +
                              "multiple of parity block size");
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("found container with offset " + cb.getOffset() +
                  ", length " + cb.getLength());
      }

      for (long offset = cb.getOffset();
           offset < cb.getOffset() + cb.getLength();
           offset += blockSize) {
        long block = offset / blockSize;
        
        int stripe = (int) (offset / parityStripeLength);

        if (stripe < 0) {
          // before the beginning of the parity file
          continue;
        }
        if (stripe >= numStripes) {
          // past the end of the parity file
          break;
        }

        parityBlocksFound++;
        
        if (stripe < startStripeIdx || stripe >= endStripeIdx) {
          continue;
        }

        if (isBlockCorrupt(cb)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("parity file for " + filePath.toString() + 
                     " corrupt in block " + block +
                     ", stripe " + stripe + "/" + numStripes);
          }
          incCorruptBlocksPerStripe(corruptBlocksPerStripe, stripe);
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug("parity file for " + filePath.toString() + 
                     " OK in block " + block +
                     ", stripe " + stripe + "/" + numStripes);
          }
        }
      }
    }

    long parityBlocksExpected = raidInfo.parityBlocksPerStripe * numStripes;
    if (parityBlocksFound != parityBlocksExpected ) {
      throw new IOException("expected " + parityBlocksExpected + 
                            " parity blocks but got " + parityBlocksFound);
    }
  }
259182
def get_block_bounds(filename):
 
 bs = []
 with open(filename,'rb') as inf:
  while True:
   bytes1 = inf.read(12)
   if len(bytes1) < 12: break
   bs.append([inf.tell()-12])
   gzip_id1,gzip_id2,compression_method,flag,mtime,xfl,osval,xlen=struct.unpack('<BBBBIBBH',bytes1)
   # ready to look in extra field
   bytes2 = inf.read(xlen) # all the extra field stuff 
   s = StringIO.StringIO(bytes2)
   obsslen = 0
   blocksize = 0
   while True:
     v1 = s.read(4)
     if len(v1) == 0: 
       break
     if len(v1) < 4:
       sys.stderr.write("lack header values ERROR\n")
       return False
     s1,s2,slen = struct.unpack('<BBH',v1)
     if s1 == 66 and s2 == 67:
       has_id = True
       obsslen = slen
       blocksize = struct.unpack('<H',s.read(slen))[0]
     else:
       v = s.read(slen)
   chunk = inf.read(blocksize-1-xlen-19)
   inf.read(9)
   bs[-1].append(inf.tell())
 return bs
1194603
private static int getBlockIdInFile(DistributedFileSystem srcFs, 
      Path srcPath, long blockId) throws IOException {
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    LocatedBlocks lbs =
        srcFs.getClient().getLocatedBlocks(srcPath.toUri().getPath(), 0, 
            srcStat.getLen());
    
    int i = 0;
    LOG.info("Look for block " + blockId + " in file " + srcPath);
    for (LocatedBlock lb: lbs.getLocatedBlocks()) {
      if (lb.getBlock().getBlockId() == blockId) {
        return i;
      }
      i++;
    }
    
    return -1;
  }
774137
static public int getBlockCrc(DataNode datanode, ReplicaToRead ri,
      int namespaceId, Block block) throws IOException {
    InputStream rawStreamIn = null;
    DataInputStream streamIn = null;
    try {
      int bytesPerCRC;
      int checksumSize;

      long crcPerBlock;

      rawStreamIn = BlockWithChecksumFileReader.getMetaDataInputStream(
          datanode.data, namespaceId, block);
      streamIn = new DataInputStream(new BufferedInputStream(rawStreamIn,
          FSConstants.BUFFER_SIZE));

      final BlockMetadataHeader header = BlockMetadataHeader
          .readHeader(streamIn);
      final DataChecksum checksum = header.getChecksum();
      if (checksum.getChecksumType() != DataChecksum.CHECKSUM_CRC32) {
        throw new IOException("File Checksum now is only supported for CRC32");
      }

      bytesPerCRC = checksum.getBytesPerChecksum();
      checksumSize = checksum.getChecksumSize();
      crcPerBlock = (((BlockWithChecksumFileReader.MetaDataInputStream) rawStreamIn)
          .getLength() - BlockMetadataHeader.getHeaderSize()) / checksumSize;

      int blockCrc = 0;
      byte[] buffer = new byte[checksumSize];
      for (int i = 0; i < crcPerBlock; i++) {
        IOUtils.readFully(streamIn, buffer, 0, buffer.length);
        int intChecksum = ((buffer[0] & 0xff) << 24)
            | ((buffer[1] & 0xff) << 16) | ((buffer[2] & 0xff) << 8)
            | ((buffer[3] & 0xff));

        if (i == 0) {
          blockCrc = intChecksum;
        } else {
          int chunkLength;
          if (i != crcPerBlock - 1 || ri.getBytesVisible() % bytesPerCRC == 0) {
            chunkLength = bytesPerCRC;
          } else {
            chunkLength = (int) ri.getBytesVisible() % bytesPerCRC;
          }
          blockCrc = CrcConcat.concatCrc(blockCrc, intChecksum, chunkLength);
        }
      }
      return blockCrc;
    } finally {
      if (streamIn != null) {
        IOUtils.closeStream(streamIn);
      }
      if (rawStreamIn != null) {
        IOUtils.closeStream(rawStreamIn);
      }
    }
  }
650963
def _split_file(f, blocksize):
    

    f_bytes = f.read()
    blocks = [int.from_bytes(f_bytes[i:i+blocksize].ljust(blocksize, b'0'), sys.byteorder) 
            for i in range(0, len(f_bytes), blocksize)]
    return len(f_bytes), blocks
529807
private CorruptFileBlocks methodBasedListCorruptFileBlocks(String path,
      String cookie) throws IOException {
    if (!namenodeProtocolProxy.isMethodSupported("listCorruptFileBlocks",
        String.class, String.class)) {
      LOG.info("NameNode version is " + namenodeVersion +
               " Using older version of getCorruptFiles.");
      if (cookie != null ) {
        return new CorruptFileBlocks(new String[0], "");
      }
      ArrayList<String> str = new ArrayList<String>();
      for (FileStatus stat : namenode.getCorruptFiles()) {
        String filename = stat.getPath().toUri().getPath();
        if (filename.startsWith(path)) {
          str.add(filename);
        }
      }
      return new CorruptFileBlocks(str.toArray(new String[str.size()]), "");
    }
    return namenode.listCorruptFileBlocks(path, cookie);
  }