Read block data from network into integer type
427780
def _read_block(blocksize, stream):
    
    blockdata = stream.read(blocksize)
    return int.from_bytes(blockdata, 'big')
238965
static void getNumDataBytesAndNumECBytesForBlockID(int numTotalBytes,
                                                     int numDataBytes,
                                                     int numRSBlocks,
                                                     int blockID,
                                                     int[] numDataBytesInBlock,
                                                     int[] numECBytesInBlock) throws WriterException {
    if (blockID >= numRSBlocks) {
      throw new WriterException("Block ID too large");
    }
    // numRsBlocksInGroup2 = 196 % 5 = 1
    int numRsBlocksInGroup2 = numTotalBytes % numRSBlocks;
    // numRsBlocksInGroup1 = 5 - 1 = 4
    int numRsBlocksInGroup1 = numRSBlocks - numRsBlocksInGroup2;
    // numTotalBytesInGroup1 = 196 / 5 = 39
    int numTotalBytesInGroup1 = numTotalBytes / numRSBlocks;
    // numTotalBytesInGroup2 = 39 + 1 = 40
    int numTotalBytesInGroup2 = numTotalBytesInGroup1 + 1;
    // numDataBytesInGroup1 = 66 / 5 = 13
    int numDataBytesInGroup1 = numDataBytes / numRSBlocks;
    // numDataBytesInGroup2 = 13 + 1 = 14
    int numDataBytesInGroup2 = numDataBytesInGroup1 + 1;
    // numEcBytesInGroup1 = 39 - 13 = 26
    int numEcBytesInGroup1 = numTotalBytesInGroup1 - numDataBytesInGroup1;
    // numEcBytesInGroup2 = 40 - 14 = 26
    int numEcBytesInGroup2 = numTotalBytesInGroup2 - numDataBytesInGroup2;
    // Sanity checks.
    // 26 = 26
    if (numEcBytesInGroup1 != numEcBytesInGroup2) {
      throw new WriterException("EC bytes mismatch");
    }
    // 5 = 4 + 1.
    if (numRSBlocks != numRsBlocksInGroup1 + numRsBlocksInGroup2) {
      throw new WriterException("RS blocks mismatch");
    }
    // 196 = (13 + 26) * 4 + (14 + 26) * 1
    if (numTotalBytes !=
        ((numDataBytesInGroup1 + numEcBytesInGroup1) *
            numRsBlocksInGroup1) +
            ((numDataBytesInGroup2 + numEcBytesInGroup2) *
                numRsBlocksInGroup2)) {
      throw new WriterException("Total bytes mismatch");
    }

    if (blockID < numRsBlocksInGroup1) {
      numDataBytesInBlock[0] = numDataBytesInGroup1;
      numECBytesInBlock[0] = numEcBytesInGroup1;
    } else {
      numDataBytesInBlock[0] = numDataBytesInGroup2;
      numECBytesInBlock[0] = numEcBytesInGroup2;
    }
  }
1120589
def read_block(self, block):
        
        if block < 0 or block > 255:
            raise ValueError("invalid block number")
        log.debug("read block {0}".format(block))
        cmd = "\x02" + chr(block) + 8 * chr(0) + self.uid
        return self.transceive(cmd)[1:9]
1142755
def read_next_block(infile, block_size=io.DEFAULT_BUFFER_SIZE):
    """"""
    chunk = infile.read(block_size)

    while chunk:
        yield chunk

        chunk = infile.read(block_size)
780485
def _read_value(self, file):
        """"""

        if self.data_type.nptype is not None:
            dtype = (np.dtype(self.data_type.nptype).newbyteorder(
                self.endianness))
            return fromfile(file, dtype=dtype, count=1)
        return self.data_type.read(file, self.endianness)
810135
def pad(data, blocksize=16):
    
    padlen = blocksize - len(data) % blocksize
    return bytes(data + bytearray(range(1, padlen)) + bytearray((padlen - 1,)))
810743
private void readNextBlock()
            throws IOException
    {
        lastReadInputCheckpoint = input.getCheckpoint();

        int control = input.read();
        if (control == -1) {
            throw new OrcCorruptionException(input.getOrcDataSourceId(), "Read past end of buffer RLE byte");
        }

        offset = 0;

        // if byte high bit is not set, this is a repetition; otherwise it is a literal sequence
        if ((control & 0x80) == 0) {
            length = control + MIN_REPEAT_SIZE;

            // read the repeated value
            int value = input.read();
            if (value == -1) {
                throw new OrcCorruptionException(input.getOrcDataSourceId(), "Reading RLE byte got EOF");
            }

            // fill buffer with the value
            Arrays.fill(buffer, 0, length, (byte) value);
        }
        else {
            // length is 2's complement of byte
            length = 0x100 - control;

            // read the literals into the buffer
            input.readFully(buffer, 0, length);
        }
    }
42898
def sendDAT(self):
        """"""
        finished = False
        blocknumber = self.context.next_block
        # Test hook
        if DELAY_BLOCK and DELAY_BLOCK == blocknumber:
            import time
            log.debug("Deliberately delaying 10 seconds...")
            time.sleep(10)
        dat = None
        blksize = self.context.getBlocksize()
        buffer = self.context.fileobj.read(blksize)
        log.debug("Read %d bytes into buffer", len(buffer))
        if len(buffer) < blksize:
            log.info("Reached EOF on file %s"
                % self.context.file_to_transfer)
            finished = True
        dat = TftpPacketDAT()
        dat.data = buffer
        dat.blocknumber = blocknumber
        self.context.metrics.bytes += len(dat.data)
        log.debug("Sending DAT packet %d", dat.blocknumber)
        self.context.sock.sendto(dat.encode().buffer,
                                 (self.context.host, self.context.tidport))
        if self.context.packethook:
            self.context.packethook(dat)
        self.context.last_pkt = dat
        return finished
386520
def _readBlock(self):
        """"""

        if self.interrupted or self.fp is None:
            if self.debug:
                log.msg('WorkerFileDownloadCommand._readBlock(): end')
            return True

        length = self.blocksize
        if self.bytes_remaining is not None and length > self.bytes_remaining:
            length = self.bytes_remaining

        if length <= 0:
            if self.stderr is None:
                self.stderr = "Maximum filesize reached, truncating file '{0}'".format(
                    self.path)
                self.rc = 1
            return True
        else:
            d = self.reader.callRemote('read', length)
            d.addCallback(self._writeData)
            return d
336476
def convert_bytes_to_ints(in_bytes, num):
    """"""
    dt = numpy.dtype('>i' + str(num))
    return numpy.frombuffer(in_bytes, dt)
1019988
public ByteBuffer readRawBlock() throws IOException, EthereumBlockReadException {
		// basically an Ethereum Block is simply a RLP encoded list
		ByteBuffer result=null;
		// get size of list
		this.in.mark(10);
		byte[] listHeader = new byte[10];
		int totalRead = 0;
		int bRead=this.in.read(listHeader);
		if (bRead == -1) {
			// no further block to read
			return result;
		} else {
			totalRead += bRead;
			while (totalRead < 10) {
				bRead=this.in.read(listHeader, totalRead, 10 - totalRead);
				if (bRead == -1) {
					throw new EthereumBlockReadException("Error: Not enough block data available: " + String.valueOf(bRead));
				}
				totalRead += bRead;
			}
		}
		ByteBuffer sizeByteBuffer=ByteBuffer.wrap(listHeader);
		long blockSize = EthereumUtil.getRLPListSize(sizeByteBuffer); // gets block size including indicator
		this.in.reset();
		// check if blockSize is valid
		if (blockSize==0) {
			throw new EthereumBlockReadException("Error: Blocksize too small");
		}
		if (blockSize<0) {
			throw new EthereumBlockReadException("Error: This block size cannot be handled currently (larger then largest number in positive signed int)");
		}
		if (blockSize>this.maxSizeEthereumBlock) {
			throw new EthereumBlockReadException("Error: Block size is larger then defined in configuration - Please increase it if this is a valid block");
		}
		// read block
		int blockSizeInt=(int)(blockSize);
		byte[] fullBlock=new byte[blockSizeInt];
		int totalByteRead=0;
		int readByte;
		while ((readByte=this.in.read(fullBlock,totalByteRead,blockSizeInt-totalByteRead))>-1) {
				totalByteRead+=readByte;
				if (totalByteRead>=blockSize) {
					break;
				}
		}
		if (totalByteRead!=blockSize) {
			 throw new EthereumBlockReadException("Error: Could not read full block");
		}
		if (!(this.useDirectBuffer)) {
		 	result=ByteBuffer.wrap(fullBlock);	
		} else {
			preAllocatedDirectByteBuffer.clear(); // clear out old bytebuffer
			preAllocatedDirectByteBuffer.limit(fullBlock.length); // limit the bytebuffer
			result=preAllocatedDirectByteBuffer;
			result.put(fullBlock);
			result.flip(); // put in read mode
		}
		result.order(ByteOrder.LITTLE_ENDIAN);	
		return result;
	}
682041
def _read_as_int(self, addr, numBytes):
        """"""
        buf = self.read_register(addr, numBytes)
        if len(buf) >= 4:
            return struct.unpack_from("<i", buf)[0]
        else:
            rtn = 0
            for i, byte in enumerate(buf):
                rtn |= byte << 8 * i
            return rtn
553758
private ByteBuffer fetchBlockByteRangeScatterGather(LocatedBlock block, 
                      long start, long len) throws IOException {
    //
    // Connect to best DataNode for desired Block, with potential offset
    //
    Socket dn = null;

    // cached block locations may have been updated by chooseDatNode()
    // or fetchBlockAt(). Always get the latest list of locations before the
    // start of the loop.
    BlockSeekContext seekContext = new BlockSeekContext(getBlockAt(
        block.getStartOffset(), false, true));
    while (true) {
      DNAddrPair retval = chooseDataNode(seekContext);
      DatanodeInfo chosenNode = retval.info;
      InetSocketAddress targetAddr = retval.addr;
      ByteBuffer result = null;
      BlockReaderLocalBase localReader = null;
      BlockReaderAccelerator remoteReader = null;

       try {
         if (DFSClient.LOG.isDebugEnabled()) {
           DFSClient.LOG.debug("fetchBlockByteRangeScatterGather " +
                    " localhst " + dfsClient.localHost +
                    " targetAddr " + targetAddr);
         }
         
         // first try reading the block locally.
         if (dfsClient.shortCircuitLocalReads && 
             NetUtils.isLocalAddressWithCaching(targetAddr.getAddress())) {
           localReader = BlockReaderLocalBase.newBlockReader(dfsClient.conf, src,
                                                dfsClient.namespaceId, block.getBlock(),
                                                chosenNode,
                                                start,
                                                len,
                                                dfsClient.metrics,
                                                verifyChecksum,
                                                this.clearOsBuffer,
                                                false);
           if (localReader != null) {
             localReader.setReadLocal(true);
             localReader.setFsStats(dfsClient.stats);
             result = localReader.readAll();
           } else if (!dfsClient.shortcircuitDisableWhenFail) {
            throw new IOException(
                "Short circuit local read not supported for this scase");
           }
         }
         if (localReader == null) {

           // go to the datanode
           dn = dfsClient.socketFactory.createSocket();
           NetUtils.connect(dn, targetAddr, dfsClient.socketTimeout,
               dfsClient.ipTosValue);
           dn.setSoTimeout(dfsClient.socketTimeout);
           remoteReader = new BlockReaderAccelerator(dfsClient.conf,
                                          targetAddr,
                                          chosenNode,
                                          dfsClient.getDataTransferProtocolVersion(),
                                          dfsClient.namespaceId, dfsClient.clientName,
                                          dn, src,
                                          block,
                                          start, len,
                                          verifyChecksum, dfsClient.metrics);
           result = remoteReader.readAll();
          }
          if (result.remaining() != len) {
            throw new IOException("truncated return from reader.read(): " +
                                "expected " + len + ", got " + 
                                  result.remaining());
          }
          if (NetUtils.isLocalAddressWithCaching(targetAddr.getAddress())) {
            dfsClient.stats.incrementLocalBytesRead(len);
            dfsClient.stats.incrementRackLocalBytesRead(len);
          } else if (dfsClient.isInLocalRack(targetAddr)) {
            dfsClient.stats.incrementRackLocalBytesRead(len);
          }

          return result;
      } catch (ChecksumException e) {
        DFSClient.LOG.warn("fetchBlockByteRangeScatterGather(). Got a checksum exception for " +
                 src + " at " + block.getBlock() + ":" +
                 e.getPos() + " from " + chosenNode.getName());
        dfsClient.reportChecksumFailure(src, block.getBlock(), chosenNode);
      } catch (IOException e) {
        DFSClient.LOG.warn("Failed to connect to " + targetAddr +
                 " for file " + src +
                 " for block " + block.getBlock().getBlockId() + ":"  +
                 StringUtils.stringifyException(e));
      } finally {
        IOUtils.closeStream(localReader);
        IOUtils.closeStream(remoteReader);
        IOUtils.closeSocket(dn);
      }
       dfsClient.incReadExpCntToStats();
      // Put chosen node into dead list, continue
      addToDeadNodes(chosenNode);
    }
  }
255382
def decode_network_values(ptype, plen, buf):
    
    nvalues = short.unpack_from(buf, header.size)[0]
    off = header.size + short.size + nvalues
    valskip = double.size

    # Check whether our expected packet size is the reported one
    assert ((valskip + 1) * nvalues + short.size + header.size) == plen
    assert double.size == number.size

    result = []
    for dstype in [ord(x) for x in buf[header.size + short.size:off]]:
        if dstype == DS_TYPE_COUNTER:
            result.append((dstype, number.unpack_from(buf, off)[0]))
            off += valskip
        elif dstype == DS_TYPE_GAUGE:
            result.append((dstype, double.unpack_from(buf, off)[0]))
            off += valskip
        elif dstype == DS_TYPE_DERIVE:
            result.append((dstype, number.unpack_from(buf, off)[0]))
            off += valskip
        elif dstype == DS_TYPE_ABSOLUTE:
            result.append((dstype, number.unpack_from(buf, off)[0]))
            off += valskip
        else:
            raise ValueError("DS type %i unsupported" % dstype)

    return result
1081699
def read_plain_int32(file_obj, count):
    """"""
    length = 4 * count
    data = file_obj.read(length)
    if len(data) != length:
        raise EOFError("Expected {} bytes but got {} bytes".format(length, len(data)))
    res = struct.unpack("<{}i".format(count).encode("utf-8"), data)
    return res
825484
def _read_data(self, count):
        """"""
        # Build a read request frame.
        frame = bytearray(count)
        frame[0] = PN532_SPI_DATAREAD
        # Send the frame and return the response, ignoring the SPI header byte.
        self._gpio.set_low(self._cs)
        self._busy_wait_ms(2)
        response = self._spi.transfer(frame)
        self._gpio.set_high(self._cs)
        return response
367703
private void readBlock(DataInputStream in,
      VersionAndOpcode versionAndOpcode, long receiverStartTime) throws IOException {
    //
    // Read in the header
    //
    long startNanoTime = System.nanoTime();
    long startTime = System.currentTimeMillis();
    
    ReadBlockHeader header = new ReadBlockHeader(versionAndOpcode);
    header.readFields(in);
    
    ReadOptions options = header.getReadOptions();
    boolean ioprioEnabled = !options.isIoprioDisabled();
    if (ioprioEnabled) {
      NativeIO.ioprioSetIfPossible(options.getIoprioClass(),
          options.getIoprioData());
    }

    int namespaceId = header.getNamespaceId();
    long blockId = header.getBlockId();
    Block block = new Block( blockId, 0 , header.getGenStamp());
    long startOffset = header.getStartOffset();
    long length = header.getLen();
    String clientName = header.getClientName();
    reuseConnection = header.getReuseConnection();
    boolean shouldProfile = header.getShouldProfile();
    FSDataNodeReadProfilingData dnData = shouldProfile ? 
        new FSDataNodeReadProfilingData() : null;

    if (shouldProfile) {
      dnData.readVersionAndOpCodeTime = (startNanoTime - receiverStartTime) ;
      dnData.readBlockHeaderTime = (System.nanoTime() - startNanoTime);
      dnData.startProfiling();
    }
    
    // send the block
    OutputStream baseStream = NetUtils.getOutputStream(s, 
        datanode.socketWriteTimeout);
    DataOutputStream out = new DataOutputStream(
                 new BufferedOutputStream(baseStream, SMALL_BUFFER_SIZE));
    
    BlockSender blockSender = null;
    String clientTraceFmt = null;
    if (ClientTraceLog.isInfoEnabled()) {    
      if (remoteAddress == null) {
        getAddresses();
      }
      clientTraceFmt = clientName.length() > 0 ? String.format(
          DN_CLIENTTRACE_FORMAT, localAddress, remoteAddress, "%d",
          "HDFS_READ", clientName, "%d",
          datanode.getDNRegistrationForNS(namespaceId).getStorageID(), block,
          "%d")
          :
          datanode.getDNRegistrationForNS(namespaceId)
          + " Served block " + block + " to " + s.getInetAddress();
    }
    updateCurrentThreadName("sending block " + block);
    InjectionHandler.processEvent(InjectionEvent.DATANODE_READ_BLOCK);
        
    try {
      try {
        blockSender = new BlockSender(namespaceId, block, startOffset, length,
            datanode.ignoreChecksumWhenRead, true, true, false,
            versionAndOpcode.getDataTransferVersion() >=
              DataTransferProtocol.PACKET_INCLUDE_VERSION_VERSION,
            false, datanode, clientTraceFmt);
        if (shouldProfile) {
          blockSender.enableReadProfiling(dnData);
        }
     } catch(IOException e) {
        sendResponse(s, (short) DataTransferProtocol.OP_STATUS_ERROR, 
            datanode.socketWriteTimeout);
        throw e;
      }
      if (ClientTraceLog.isInfoEnabled()) {
        ClientTraceLog.info("Sending blocks. namespaceId: "
            + namespaceId + " block: " + block + " to " + remoteAddress);
      }

      out.writeShort(DataTransferProtocol.OP_STATUS_SUCCESS); // send op status
      if (shouldProfile) {
        dnData.startSendBlock();
      }
      
      long read = blockSender.sendBlock(out, baseStream, null); // send data
      if (shouldProfile) {
        dnData.endSendBlock();
      }
      
      // report finalization information and block length.
      ReplicaToRead replicaRead = blockSender.getReplicaToRead();
      if (replicaRead == null) {
        replicaRead = datanode.data.getReplicaToRead(namespaceId, block);
      }
      if (replicaRead == null) {
        throw new IOException("Can't find block " + block + " in volumeMap");
      }
      
      int fadvise = header.getReadOptions().getFadvise();
      if (fadvise != 0) {
        blockSender.fadviseStream(fadvise, startOffset, length);
      }

      boolean isBlockFinalized = replicaRead.isFinalized();
      long blockLength = replicaRead.getBytesVisible();
      out.writeBoolean(isBlockFinalized);
      out.writeLong(blockLength);
      if (shouldProfile) {
        dnData.write(out);
      }
      out.flush();

      if (blockSender.didSendEntireByteRange()) {
        // See if client verification succeeded. 
        // This is an optional response from client.
        try {
          short status = in.readShort();
          if ((status == DataTransferProtocol.OP_STATUS_CHECKSUM_OK ||
               status == DataTransferProtocol.OP_STATUS_SUCCESS) && 
              datanode.blockScanner != null) {
            datanode.blockScanner.verifiedByClient(namespaceId, block);
          }
        } catch (IOException ignored) {}
      }
      
      long readDuration = System.currentTimeMillis() - startTime;
      
      if (readDuration > datanode.thresholdLogReadDuration
          && !ClientTraceLog.isInfoEnabled()) {
        // No need to double logging if we've logged duration of every read
        
        if (remoteAddress == null) {
            getAddresses();
        }
        LOG.info("Sent block " + block + " to " + remoteAddress + " offset "
            + startOffset + " len " + length + " duration " + readDuration
            + " (longer than " + datanode.thresholdLogReadDuration + ")");
      }
      
      datanode.myMetrics.bytesReadLatency.inc(readDuration);
      datanode.myMetrics.bytesRead.inc((int) read);
      if (read > KB_RIGHT_SHIFT_MIN) {
        datanode.myMetrics.bytesReadRate.inc((int) (read >> KB_RIGHT_SHIFT_BITS),
                              readDuration);
      }
      datanode.myMetrics.blocksRead.inc();
    } catch ( SocketException ignored ) {
      // Its ok for remote side to close the connection anytime.
      datanode.myMetrics.blocksRead.inc();
      IOUtils.closeStream(out);
      
      // log exception to debug exceptions like socket timeout exceptions
      if (LOG.isDebugEnabled()) {
        LOG.debug("Ignore exception while sending blocks. namespaceId: "
          + namespaceId + " block: " + block + " to " + remoteAddress + ": "
          + ignored.getMessage());
      }
    } catch ( IOException ioe ) {
      /* What exactly should we do here?
       * Earlier version shutdown() datanode if there is disk error.
       */
      LOG.warn(datanode.getDatanodeInfo() +  ":Got exception while serving " + 
          "namespaceId: " + namespaceId + " block: " + block + " to " +
                s.getInetAddress() + ", client: " + clientName + ":\n" + 
                StringUtils.stringifyException(ioe) );
      throw ioe;
    } finally {
      IOUtils.closeStream(blockSender);
    }
  }