Compute a compressed integer buffer
547465
private byte[] bufX( long bias, int scale, int off, int log ) {
    byte[] bs = MemoryManager.malloc1((_len <<log)+off);
    int j = 0;
    for( int i=0; i< _len; i++ ) {
      long le = -bias;
      if(_id == null || _id.length == 0 || (j < _id.length && _id[j] == i)){
        if( isNA2(j) ) {
          le = NAS[log];
        } else {
          int x = (_xs.get(j)==Integer.MIN_VALUE+1 ? 0 : _xs.get(j))-scale;
          le += x >= 0
              ? _ms.get(j)*PrettyPrint.pow10i( x)
              : _ms.get(j)/PrettyPrint.pow10i(-x);
        }
        ++j;
      }
      switch( log ) {
      case 0:          bs [i    +off] = (byte)le ; break;
      case 1: UnsafeUtils.set2(bs,(i<<1)+off,  (short)le); break;
      case 2: UnsafeUtils.set4(bs, (i << 2) + off, (int) le); break;
      case 3: UnsafeUtils.set8(bs, (i << 3) + off, le); break;
      default: throw H2O.fail();
      }
    }
    assert j == _sparseLen :"j = " + j + ", _sparseLen = " + _sparseLen;
    return bs;
  }
797053
void compressOffsetArray(int i0) {
		int i_ = i0;
		while (m_offsetPts.get(i_).m_prev < i_)
			i_ = m_offsetPts.get(i_).m_prev;

		int j = 0, i = i_;

		do {
			GraphicPoint pt = m_offsetPts.get(i);
			m_offsetPts.set(j, pt);
			i = pt.m_next;
			j++;
		} while (i != i_);

		m_offsetPts.set(j, m_offsetPts.get(0)); // duplicate closing point

		m_offsetPtCount = j + 1;
	}
888542
function (num, totalBytes) {
  var numBytes = Math.floor(Math.log(num) / Math.log(0xff)) + 1;
  var buffer = new Buffer(totalBytes);

  buffer.fill(0);
  buffer.writeUIntBE(num, totalBytes - numBytes, numBytes);

  return buffer;
}
328590
private void readCompressedIntegerRowByNumber( ByteBuffer rowdata, int rn, long[] adrows, RandomAccessFile thefile )
            throws IOException, DataFormatException {
        int offset = (int) (adrows[rn + 1] - adrows[rn]);

        thefile.seek(adrows[rn]);
        /*
         * Read how many bytes the values are ex 1 => if encoded: 1 byte for the value and one byte
         * for the count = 2 2 => if encoded: 2 bytes for the value and one byte for the count = 3
         * etc... etc
         */
        int bytespervalue = (thefile.read() & 0xff);
        ByteBuffer cell = ByteBuffer.allocate(bytespervalue);
        int cellValue = 0;

        /* Create the buffer in which read the compressed row */
        byte[] tmp = new byte[offset - 1];
        thefile.read(tmp);
        ByteBuffer tmpBuffer = ByteBuffer.wrap(tmp);
        tmpBuffer.order(ByteOrder.nativeOrder());

        /*
         * Create the buffer in which read the decompressed row. The final decompressed row will
         * always contain 4-byte integer values
         */
        if ((offset - 1) == (bytespervalue * fileWindow.getCols())) {
            /* There is no compression in this row */
            for( int i = 0; i < offset - 1; i = i + bytespervalue ) {
                /* Read the value */
                tmpBuffer.get(cell.array());

                /*
                 * Integers can be of 1, 2, or 4 bytes. As rasterBuffer expects 4 byte integers we
                 * need to pad them with 0's. The order of the padding is determined by the
                 * ByteOrder of the buffer.
                 */
                if (bytespervalue == 1) {
                    cellValue = (cell.get(0) & 0xff);
                } else if (bytespervalue == 2) {
                    cellValue = cell.getShort(0);
                } else if (bytespervalue == 4) {
                    cellValue = cell.getInt(0);
                }
                // if (logger.isDebugEnabled()) logger.debug("tmpint=" + tmpint
                // );
                rowdata.putInt(cellValue);
            }
        } else {
            /*
             * If the row is compressed, then the values appear in pairs (like couples a party). The
             * couple is composed of the count and the value value (WARNING: this can be more than
             * one byte). Therefore, knowing the length of the compressed row we can calculate the
             * number of couples.
             */
            int couples = (offset - 1) / (1 + bytespervalue);

            for( int i = 0; i < couples; i++ ) {
                /* Read the count of values */
                int count = (tmpBuffer.get() & 0xff);

                /* Read the value */
                tmpBuffer.get(cell.array());

                /*
                 * Integers can be of 1, 2, or 4 bytes. As rasterBuffer expects 4 byte integers we
                 * need to pad them with 0's. The order of the padding is determined by the
                 * ByteOrder of the buffer.
                 */
                if (bytespervalue == 1) {
                    cellValue = (cell.get(0) & 0xff);
                } else if (bytespervalue == 2) {
                    cellValue = cell.getShort(0);
                } else if (bytespervalue == 4) {
                    cellValue = cell.getInt(0);
                }
                /*
                 * Now write the cell value the required number of times to the raster row data
                 * buffer.
                 */
                for( int j = 0; j < count; j++ ) {
                    // // if (logger.isDebugEnabled()) logger.debug(" " +
                    // tmpint);
                    rowdata.putInt(cellValue);
                }
            }
        }
    }
1373615
def compressed_bytes2ibytes(compressed, size):
    
    decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
    for i in range(0, mo_math.ceiling(len(compressed), size), size):
        try:
            block = compressed[i: i + size]
            yield decompressor.decompress(block)
        except Exception as e:
            Log.error("Not expected", e)
217946
def read32(bytestream):
  """"""
  dt = np.dtype(np.uint32).newbyteorder('>')
  return np.frombuffer(bytestream.read(4), dtype=dt)[0]
572622
private byte[] bufD(final int valsz){
    int log = 0;
    while((1 << log) < valsz)++log;
    assert (1 << log) == valsz;
    final int ridsz = _len >= 65535?4:2;
    final int elmsz = ridsz + valsz;
    int off = CXDChunk.OFF;
    byte [] buf = MemoryManager.malloc1(off + _sparseLen*elmsz,true);
    for( int i=0; i<_sparseLen; i++, off += elmsz ) {
      if(ridsz == 2)
        UDP.set2(buf,off,(short)_id[i]);
      else
        UDP.set4(buf,off,_id[i]);
      final double dval = _ds == null?isNA2(i)?Double.NaN:_ls[i]*PrettyPrint.pow10(_xs[i]):_ds[i];
      switch(valsz){
        case 4:
          UDP.set4f(buf, off + ridsz, (float) dval);
          break;
        case 8:
          UDP.set8d(buf, off + ridsz, dval);
          break;
        default:
          throw H2O.unimpl();
      }
    }
    assert off==buf.length;
    return buf;
  }
294293
def load_uint_b(buffer, width):
    
    result = 0
    for idx in range(width):
        result += buffer[idx] << (8 * idx)
    return result
119062
static long toInt(final byte[] buffer, final int initialOffset) {
        int offset = initialOffset;
        return (buffer[offset] << 24)
                + ((buffer[++offset] & 0xFF) << 16)
                + ((buffer[++offset] & 0xFF) << 8)
                + (buffer[++offset] & 0xFF);
    }
510226
def compress(
      level = Zlib::DEFAULT_COMPRESSION,
      window_bits = Zlib::MAX_WBITS,
      mem_level = Zlib::DEF_MEM_LEVEL,
      strategy = Zlib::DEFAULT_STRATEGY
    )
      wrap_with_rewind(@compressed_data, @filtered_data) do
        z = Zlib::Deflate.new level, window_bits, mem_level, strategy
        until @filtered_data.eof? do
          buffer_size = 2 ** 16
          flush = Zlib::NO_FLUSH
          flush = Zlib::FINISH if @filtered_data.size - @filtered_data.pos < buffer_size
          @compressed_data << z.deflate(@filtered_data.read(buffer_size), flush)
        end
        z.finish
        z.close
        truncate_io @compressed_data
      end
      @is_compressed_data_modified = false
      self
    end
650356
private ByteBuf tryCompress(ByteBuf buf) {
        final int uncompressedLength = buf.readableBytes();
        final byte[] compressed = new byte[Snappy.maxCompressedLength(uncompressedLength)];
        final int compressedLength;

        if (buf.hasArray()) {
            // Save a memory copy by reading directly from the backing array.
            compressedLength = Snappy.compress(
                    buf.array(), buf.arrayOffset() + buf.readerIndex(), uncompressedLength, compressed, 0);
        } else {
            final byte[] data = new byte[buf.readableBytes()];
            buf.getBytes(buf.readerIndex(), data);
            compressedLength = Snappy.compress(data, 0, data.length, compressed, 0);
        }

        final double ratio = (double) compressedLength / uncompressedLength;
        if (ratio > minCompressionRatio) {
            // Compression doesn't make enough of a difference.
            return null;
        }

        // Save a memory copy by deferring trimming until we know the result will be used.
        final byte[] trimmedCompressed = Arrays.copyOf(compressed, compressedLength);
        return Unpooled.wrappedBuffer(trimmedCompressed);
    }
1077673
def nbytes_stored(self):
        """"""
        m = getsize(self._store, self._path)
        if self._chunk_store is None:
            return m
        else:
            n = getsize(self._chunk_store, self._path)
            if m < 0 or n < 0:
                return -1
            else:
                return m + n
370744
def UInt32():
  """"""
  with _mutex:
    try:
      return _random_buffer.pop()
    except IndexError:
      data = os.urandom(struct.calcsize("=L") * _random_buffer_size)
      _random_buffer.extend(
          struct.unpack("=" + "L" * _random_buffer_size, data))
      return _random_buffer.pop()
1160506
static void lowLevelUncompressPairs(
      final int[] pairArray,           // output
      final int numPairsToDecode,      // input, size of output, must be int
      final int numBaseBits,           // input, cannot exceed 6 bits
      final int[] compressedWords,     // input
      final long numCompressedWords) { // input

    int pairIndex = 0;

    final long[] ptrArr = new long[3];
    int nextWordIndex = 0;
    long bitBuf = 0;
    int bufBits = 0;

    final long golombLoMask = (1L << numBaseBits) - 1L;

    int predictedRowIndex = 0;
    int predictedColIndex = 0;

    // for each pair we need to read:
    // xDelta (12-bit length-limited unary)
    // yDeltaHi (unary)
    // yDeltaLo (basebits)

    for (pairIndex = 0; pairIndex < numPairsToDecode; pairIndex++) {

      //MAYBE_FILL_BITBUF(compressedWords,wordIndex,12); // ensure 12 bits in bit buffer
      if (bufBits < 12) { // Prepare for a 12-bit peek into the bitstream.
        bitBuf |= ((compressedWords[nextWordIndex++] & 0XFFFF_FFFFL) << bufBits);
        bufBits += 32;
      }

      final int peek12 = (int) (bitBuf & 0XFFFL);
      final int lookup = lengthLimitedUnaryDecodingTable65[peek12] & 0XFFFF;
      final int codeWordLength = lookup >>> 8;
      final int xDelta = lookup & 0XFF;
      bitBuf >>>= codeWordLength;
      bufBits -= codeWordLength;

      //TODO Inline ReadUnary
      ptrArr[NEXT_WORD_IDX] = nextWordIndex;
      ptrArr[BIT_BUF] = bitBuf;
      ptrArr[BUF_BITS] = bufBits;
      assert nextWordIndex == ptrArr[NEXT_WORD_IDX]; //catch sign extension error

      final long golombHi = readUnary(compressedWords, ptrArr);

      nextWordIndex = (int) ptrArr[NEXT_WORD_IDX];
      bitBuf = ptrArr[BIT_BUF];
      bufBits = (int) ptrArr[BUF_BITS];
      assert nextWordIndex == ptrArr[NEXT_WORD_IDX]; //catch truncation error
      //END Inline ReadUnary

      //MAYBE_FILL_BITBUF(compressedWords,wordIndex,numBaseBits); // ensure numBaseBits in bit buffer
      if (bufBits < numBaseBits) { // Prepare for a numBaseBits peek into the bitstream.
        bitBuf |= ((compressedWords[nextWordIndex++] & 0XFFFF_FFFFL) << bufBits);
        bufBits += 32;
      }

      final long golombLo = bitBuf & golombLoMask;
      bitBuf >>>= numBaseBits;
      bufBits -= numBaseBits;
      final long yDelta = (golombHi << numBaseBits) | golombLo;

      // Now that we have yDelta and xDelta, we can compute the pair's row and column.
      if (yDelta > 0) { predictedColIndex = 0; }
      final int rowIndex = predictedRowIndex + (int) yDelta;
      final int colIndex = predictedColIndex + xDelta;
      final int rowCol = (rowIndex << 6) | colIndex;
      pairArray[pairIndex] = rowCol;
      predictedRowIndex = rowIndex;
      predictedColIndex = colIndex + 1;
    }
    // check for buffer over-run
    assert (nextWordIndex <= numCompressedWords)
      : "nextWdIdx: " + nextWordIndex + ", #CompWds: " + numCompressedWords;
  }
389910
@Override
    synchronized public int encodeIntoCompressedByteBuffer(
            final ByteBuffer targetBuffer,
            final int compressionLevel) {
        int neededCapacity = getNeededByteBufferCapacity(countsArrayLength);
        if (intermediateUncompressedByteBuffer == null || intermediateUncompressedByteBuffer.capacity() < neededCapacity) {
            intermediateUncompressedByteBuffer = ByteBuffer.allocate(neededCapacity).order(BIG_ENDIAN);
        }
        intermediateUncompressedByteBuffer.clear();
        int initialTargetPosition = targetBuffer.position();

        final int uncompressedLength = encodeIntoByteBuffer(intermediateUncompressedByteBuffer);
        targetBuffer.putInt(getCompressedEncodingCookie());

        targetBuffer.putInt(0); // Placeholder for compressed contents length

        Deflater compressor = new Deflater(compressionLevel);
        compressor.setInput(intermediateUncompressedByteBuffer.array(), 0, uncompressedLength);
        compressor.finish();

        byte[] targetArray;

        if (targetBuffer.hasArray()) {
            targetArray = targetBuffer.array();
        } else {
            if (intermediateUncompressedByteArray == null ||
                intermediateUncompressedByteArray.length < targetBuffer.capacity()) {
                intermediateUncompressedByteArray = new byte[targetBuffer.capacity()];
            }
            targetArray = intermediateUncompressedByteArray;
        }

        int compressedTargetOffset = initialTargetPosition + 8;
        int compressedDataLength =
            compressor.deflate(
                targetArray,
                compressedTargetOffset,
                targetArray.length - compressedTargetOffset
            );
        compressor.end();

        if (!targetBuffer.hasArray()) {
            targetBuffer.put(targetArray, compressedTargetOffset, compressedDataLength);
        }

        targetBuffer.putInt(initialTargetPosition + 4, compressedDataLength); // Record the compressed length
        int bytesWritten = compressedDataLength + 8;
        targetBuffer.position(initialTargetPosition + bytesWritten);
        return bytesWritten;
    }
1331888
@Override
    synchronized public int encodeIntoCompressedByteBuffer(
            final ByteBuffer targetBuffer,
            final int compressionLevel) {
        targetBuffer.putInt(DHIST_compressedEncodingCookie);
        targetBuffer.putInt(getNumberOfSignificantValueDigits());
        targetBuffer.putLong(configuredHighestToLowestValueRatio);
        return integerValuesHistogram.encodeIntoCompressedByteBuffer(targetBuffer, compressionLevel) + 16;
    }
38726
@SuppressWarnings("WeakerAccess")
    public static AVLTreeDigest fromBytes(ByteBuffer buf) {
        int encoding = buf.getInt();
        if (encoding == VERBOSE_ENCODING) {
            double min = buf.getDouble();
            double max = buf.getDouble();
            double compression = buf.getDouble();
            AVLTreeDigest r = new AVLTreeDigest(compression);
            r.setMinMax(min, max);
            int n = buf.getInt();
            double[] means = new double[n];
            for (int i = 0; i < n; i++) {
                means[i] = buf.getDouble();
            }
            for (int i = 0; i < n; i++) {
                r.add(means[i], buf.getInt());
            }
            return r;
        } else if (encoding == SMALL_ENCODING) {
            double min = buf.getDouble();
            double max = buf.getDouble();
            double compression = buf.getDouble();
            AVLTreeDigest r = new AVLTreeDigest(compression);
            r.setMinMax(min, max);
            int n = buf.getInt();
            double[] means = new double[n];
            double x = 0;
            for (int i = 0; i < n; i++) {
                double delta = buf.getFloat();
                x += delta;
                means[i] = x;
            }

            for (int i = 0; i < n; i++) {
                int z = decode(buf);
                r.add(means[i], z);
            }
            return r;
        } else {
            throw new IllegalStateException("Invalid format for serialized histogram");
        }
    }