Reads and handles NBCROW stream.

        This stream contains list of values of one returned row in a compressed way,
        introduced in TDS 7.3.B
        Stream format url: http://msdn.microsoft.com/en-us/library/dd304783.aspx
556010
def process_nbcrow(self):
        
        self.log_response_message("got NBCROW message")
        r = self._reader
        info = self.res_info
        if not info:
            self.bad_stream('got row without info')
        assert len(info.columns) > 0
        info.row_count += 1

        # reading bitarray for nulls, 1 represent null values for
        # corresponding fields
        nbc = readall(r, (len(info.columns) + 7) // 8)
        for i, curcol in enumerate(info.columns):
            if tds_base.my_ord(nbc[i // 8]) & (1 << (i % 8)):
                value = None
            else:
                value = curcol.serializer.read(r)
            self.row[i] = value
126798
def decode_streaming(self, data):
        
        # Reverse lookup table: map (bitsize, value) to symbols
        lookup = dict(((b, v), s) for (s, (b, v)) in self._table.items())

        buffer = 0
        size = 0
        for byte in data:
            for m in [128, 64, 32, 16, 8, 4, 2, 1]:
                buffer = (buffer << 1) + bool(from_byte(byte) & m)
                size += 1
                if (size, buffer) in lookup:
                    symbol = lookup[size, buffer]
                    if symbol == _EOF:
                        return
                    yield symbol
                    buffer = 0
                    size = 0
348940
def bread(stream):
    
    if hasattr(stream, "read"):
        return bdecode(stream.read())
    else:
        handle = open(stream, "rb")
        try:
            return bdecode(handle.read())
        finally:
            handle.close()
318307
def ParseFileDownloadedRow(
      self, parser_mediator, query, row, **unused_kwargs):
    
    query_hash = hash(query)

    event_data = ChromeHistoryFileDownloadedEventData()
    event_data.full_path = self._GetRowValue(query_hash, row, 'target_path')
    event_data.offset = self._GetRowValue(query_hash, row, 'id')
    event_data.query = query
    event_data.received_bytes = self._GetRowValue(
        query_hash, row, 'received_bytes')
    event_data.total_bytes = self._GetRowValue(query_hash, row, 'total_bytes')
    event_data.url = self._GetRowValue(query_hash, row, 'url')

    timestamp = self._GetRowValue(query_hash, row, 'start_time')
    date_time = dfdatetime_webkit_time.WebKitTime(timestamp=timestamp)
    event = time_events.DateTimeValuesEvent(
        date_time, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED)
    parser_mediator.ProduceEventWithEventData(event, event_data)
817064
private void parseQueryRows(boolean lastChunk) {
        while (true) {
            int openBracketPos = findNextChar(responseContent, '{');
            if (isEmptySection(openBracketPos) || (lastChunk && openBracketPos < 0)) {
                sectionDone();
                queryParsingState = transitionToNextToken(lastChunk);
                queryRowClosingPositionProcessor = null;
                queryRowClosingProcessorIndex = 0;
                break;
            }

            if (queryRowClosingPositionProcessor == null) {
                queryRowClosingPositionProcessor = new ClosingPositionBufProcessor('{', '}', true);
                queryRowClosingProcessorIndex = responseContent.readerIndex();
            }

            int lengthToScan = responseContent.writerIndex() - this.queryRowClosingProcessorIndex;
            int closeBracketPos = responseContent.forEachByte(queryRowClosingProcessorIndex,
                lengthToScan, queryRowClosingPositionProcessor);

            if (closeBracketPos == -1) {
                queryRowClosingProcessorIndex = responseContent.writerIndex();
                break;
            }

            queryRowClosingPositionProcessor = null;
            queryRowClosingProcessorIndex = 0;
            int length = closeBracketPos - openBracketPos - responseContent.readerIndex() + 1;
            responseContent.skipBytes(openBracketPos);
            ByteBuf resultSlice = responseContent.readSlice(length);
            queryRowObservable.onNext(resultSlice.copy());
            responseContent.discardSomeReadBytes();
        }
    }
291687
def process_row(self):
        
        self.log_response_message("got ROW message")
        r = self._reader
        info = self.res_info
        info.row_count += 1
        for i, curcol in enumerate(info.columns):
            curcol.value = self.row[i] = curcol.serializer.read(r)
1035438
private void read(InputStream is, Table table) throws IOException
   {
      byte[] buffer = new byte[m_definition.getPageSize()];
      while (true)
      {
         int bytesRead = is.read(buffer);
         if (bytesRead == -1)
         {
            break;
         }

         if (bytesRead != buffer.length)
         {
            throw new IOException("Unexpected end of file");
         }

         readPage(buffer, table);
      }
   }
802020
def read_raw_table(self, table):
        
        fields = self.table_relations(table) if self.cast else None
        field_names = [f.name for f in self.table_relations(table)]
        field_len = len(field_names)
        table_path = os.path.join(self.root, table)
        with _open_table(table_path, self.encoding) as tbl:
            for line in tbl:
                cols = decode_row(line, fields=fields)
                if len(cols) != field_len:
                    # should this throw an exception instead?
                    logging.error('Number of stored fields ({}) '
                                  'differ from the expected number({}); '
                                  'fields may be misaligned!'
                                  .format(len(cols), field_len))
                row = OrderedDict(zip(field_names, cols))
                yield row
262762
public List<String> nextRow() throws IOException {
      row = new ArrayList<>();
      STATE = START;
      int c;
      int readCount = 0;
      gobbleWhiteSpace();
      while ((c = read()) != -1) {
         if (c == '\r') {
            if (bufferPeek() == '\n') {
               continue;
            } else {
               c = '\n';
            }
         }
         readCount++;
         switch (STATE) {
            case START:
               STATE = beginOfLine(c);
               break;
            case IN_QUOTE:
               wasQuoted = true;
               STATE = inField(c, true);
               break;
            case IN_FIELD:
               STATE = inField(c, false);
               break;
            case OUT_QUOTE:
               STATE = outQuote(c);
               break;
            default:
               throw new IOException("State [" + STATE + "]");
         }
         if (STATE == END_OF_ROW) {
            break;
         }
      }
      if (readCount > 0) {
         addCell(wasQuoted);
      }
      if (row.isEmpty()) {
         return null;
      }
      return new ArrayList<>(row);
   }
1310099
async def read(self, n=None):
        
        if self._streamed:
            return b''
        buffer = []
        async for body in self:
            buffer.append(body)
        return b''.join(buffer)
923693
private void readCompressedFPRowByNumber( ByteBuffer rowdata, int rn, long[] adrows, RandomAccessFile thefile, int typeBytes )
            throws DataFormatException, IOException {
        int offset = (int) (adrows[rn + 1] - adrows[rn]);
        /*
         * The fact that the file is compressed does not mean that the row is compressed. If the
         * first byte is 0 (49), then the row is compressed, otherwise (first byte = 48) the row has
         * to be read in simple XDR uncompressed format.
         */
        byte[] tmp = new byte[offset - 1];
        thefile.seek(adrows[rn]);
        int firstbyte = (thefile.read() & 0xff);
        if (firstbyte == 49) {
            /* The row is compressed. */
            // thefile.seek((long) adrows[rn] + 1);
            thefile.read(tmp, 0, offset - 1);
            Inflater decompresser = new Inflater();
            decompresser.setInput(tmp, 0, tmp.length);
            decompresser.inflate(rowdata.array());
            decompresser.end();
        } else if (firstbyte == 48) {
            /* The row is NOT compressed */
            // thefile.seek((long) (adrows[rn]));
            // if (thefile.read() == 48)
            // {
            // thefile.seek((long) (adrows[rn] + 1));
            thefile.read(rowdata.array(), 0, offset - 1);
            // }
        }
    }
684149
def parse_buffer(stream, separator=None):
    
    rows = []
    lines = []
    for line, row in parse_lines(stream, separator):
        lines.append(line)
        rows.append(row)
    cols = zip(*rows)
    return {
        'rows': rows,
        'lines': lines,
        'cols': cols,
        }
488163
def format_results(self, rows):
        
        needs_utc = _DJANGO_VERSION >= 14 and settings.USE_TZ
        if not (needs_utc or not self.driver_supports_utf8):
            return tuple(rows)
        # FreeTDS (and other ODBC drivers?) don't support Unicode yet, so we
        # need to decode UTF-8 data coming from the DB
        fr = []
        for row in rows:
            if not self.driver_supports_utf8 and isinstance(row, binary_type):
                row = row.decode(self.encoding)

            elif needs_utc and isinstance(row, datetime.datetime):
                row = row.replace(tzinfo=timezone.utc)
            fr.append(row)
        return tuple(fr)
763755
def decode_timeseries_row(self, tsrow, tscols=None,
                              convert_timestamp=False):
        
        row = []
        for i, cell in enumerate(tsrow.cells):
            col = None
            if tscols is not None:
                col = tscols[i]
            if cell.HasField('varchar_value'):
                if col and not (col.type == TsColumnType.Value('VARCHAR') or
                                col.type == TsColumnType.Value('BLOB')):
                    raise TypeError('expected VARCHAR or BLOB column')
                else:
                    row.append(cell.varchar_value)
            elif cell.HasField('sint64_value'):
                if col and col.type != TsColumnType.Value('SINT64'):
                    raise TypeError('expected SINT64 column')
                else:
                    row.append(cell.sint64_value)
            elif cell.HasField('double_value'):
                if col and col.type != TsColumnType.Value('DOUBLE'):
                    raise TypeError('expected DOUBLE column')
                else:
                    row.append(cell.double_value)
            elif cell.HasField('timestamp_value'):
                if col and col.type != TsColumnType.Value('TIMESTAMP'):
                    raise TypeError('expected TIMESTAMP column')
                else:
                    dt = cell.timestamp_value
                    if convert_timestamp:
                        dt = datetime_from_unix_time_millis(
                            cell.timestamp_value)
                    row.append(dt)
            elif cell.HasField('boolean_value'):
                if col and col.type != TsColumnType.Value('BOOLEAN'):
                    raise TypeError('expected BOOLEAN column')
                else:
                    row.append(cell.boolean_value)
            else:
                row.append(None)
        return row
732499
private List<FileData> doFileExtract(RowBatch rowBatch) {
        List<FileData> fileDatas = new ArrayList<FileData>();
        // 处理数据
        Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
        List<EventData> eventDatas = rowBatch.getDatas();
        for (EventData eventData : eventDatas) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }

            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline,
                                                                                         eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                           + " dataMediaPair is null,please check");
            }

            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (dataMediaPair.getResolverData() == null
                    || dataMediaPair.getResolverData().getExtensionDataType() == null
                    || (dataMediaPair.getResolverData().getExtensionDataType().isClazz() && StringUtils.isBlank(dataMediaPair.getResolverData().getClazzPath()))
                    || (dataMediaPair.getResolverData().getExtensionDataType().isSource() && StringUtils.isBlank(dataMediaPair.getResolverData().getSourceText()))) {
                    continue;
                }

                FileResolver fileResolver = null;

                if (dataMediaPair.getResolverData() != null) {
                    fileResolver = extensionFactory.getExtension(FileResolver.class, dataMediaPair.getResolverData());
                } else {
                    continue;
                }

                if (fileResolver == null) {
                    throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                               + " the fileResolver className  = "
                                               + dataMediaPair.getResolverData().getClazzPath()
                                               + " is null ,please check the class");
                }

                if (fileResolver instanceof RemoteDirectoryFetcherAware) {
                    RemoteDirectoryFetcherAware remoteDirectoryFetcherAware = (RemoteDirectoryFetcherAware) fileResolver;
                    remoteDirectoryFetcherAware.setRemoteDirectoryFetcher(arandaRemoteDirectoryFetcher);
                }

                List<FileData> singleRowFileDatas = getSingleRowFileInfos(dataMediaPair.getId(), fileResolver,
                                                                          eventData);
                // 做一下去重处理
                for (FileData data : singleRowFileDatas) {
                    if (!fileDatas.contains(data)) {
                        fileDatas.add(data);
                    }
                }
            }
        }

        // 判断是否需要进行图片重复同步检查
        if (pipeline.getParameters().getFileDetect()) {
            doFileDetectCollector(pipeline, fileDatas);
        }
        return fileDatas;
    }
681848
function(chunk)
  {
    // get all the rows in the current chunk that overlap with the requested
    // window
    var chunkStart = chunk.getStartIndex();
    var chunkEnd   = chunk.getEndIndex();
    var rows       = chunk.getRows().slice(
        Math.max(chunkStart, start) - chunkStart,
        Math.min(chunkEnd, end) + 1 - chunkStart);

    var rowIndex = 0;
    var rowsLength = rows.length;

    // create a function that can be called to batch-process rows
    var processRows = function()
    {
      // get the start position and start time
      var startIndex = rowIndex;
      var startTime  = Date.now();

      var each = options.each;
      while (rowIndex < rowsLength)
      {
        // invoke the each() callback on the current row
        var ret = each(rows[rowIndex++]);

        context.numRowsProcessed++;

        // if the callback returned false, stop processing rows
        if (ret === false)
        {
          var stoppedProcessingRows = true;
          break;
        }

        // use the current position and current time to check if we've been
        // processing rows for too long; if so, leave the rest for the next
        // tick of the event loop
        if ((rowIndex - startIndex) >= context.rowBatchSize &&
            (Date.now() - startTime) > context.rowBatchDuration)
        {
          process.nextTick(processRows);
          break;
        }
      }

      // if there are no more rows for us to process in this chunk
      if (!(rowIndex < rowsLength) || stoppedProcessingRows)
      {
        // if we exhausted all the rows in this chunk and we haven't yet
        // processed all the rows we want to process, ask the result stream to
        // do another read
        if (!(rowIndex < rowsLength) &&
            context.numRowsProcessed !== context.maxNumRowsToProcess)
        {
          resultStream.read();
        }
        else
        {
          // we've either processed all the rows we wanted to process or we
          // were told to stop processing rows by the each() callback; either
          // way, close the result stream to complete the operation
          resultStream.asyncClose();
        }
      }
    };

    // start processing rows
    processRows();
  }
328590
private void readCompressedIntegerRowByNumber( ByteBuffer rowdata, int rn, long[] adrows, RandomAccessFile thefile )
            throws IOException, DataFormatException {
        int offset = (int) (adrows[rn + 1] - adrows[rn]);

        thefile.seek(adrows[rn]);
        /*
         * Read how many bytes the values are ex 1 => if encoded: 1 byte for the value and one byte
         * for the count = 2 2 => if encoded: 2 bytes for the value and one byte for the count = 3
         * etc... etc
         */
        int bytespervalue = (thefile.read() & 0xff);
        ByteBuffer cell = ByteBuffer.allocate(bytespervalue);
        int cellValue = 0;

        /* Create the buffer in which read the compressed row */
        byte[] tmp = new byte[offset - 1];
        thefile.read(tmp);
        ByteBuffer tmpBuffer = ByteBuffer.wrap(tmp);
        tmpBuffer.order(ByteOrder.nativeOrder());

        /*
         * Create the buffer in which read the decompressed row. The final decompressed row will
         * always contain 4-byte integer values
         */
        if ((offset - 1) == (bytespervalue * fileWindow.getCols())) {
            /* There is no compression in this row */
            for( int i = 0; i < offset - 1; i = i + bytespervalue ) {
                /* Read the value */
                tmpBuffer.get(cell.array());

                /*
                 * Integers can be of 1, 2, or 4 bytes. As rasterBuffer expects 4 byte integers we
                 * need to pad them with 0's. The order of the padding is determined by the
                 * ByteOrder of the buffer.
                 */
                if (bytespervalue == 1) {
                    cellValue = (cell.get(0) & 0xff);
                } else if (bytespervalue == 2) {
                    cellValue = cell.getShort(0);
                } else if (bytespervalue == 4) {
                    cellValue = cell.getInt(0);
                }
                // if (logger.isDebugEnabled()) logger.debug("tmpint=" + tmpint
                // );
                rowdata.putInt(cellValue);
            }
        } else {
            /*
             * If the row is compressed, then the values appear in pairs (like couples a party). The
             * couple is composed of the count and the value value (WARNING: this can be more than
             * one byte). Therefore, knowing the length of the compressed row we can calculate the
             * number of couples.
             */
            int couples = (offset - 1) / (1 + bytespervalue);

            for( int i = 0; i < couples; i++ ) {
                /* Read the count of values */
                int count = (tmpBuffer.get() & 0xff);

                /* Read the value */
                tmpBuffer.get(cell.array());

                /*
                 * Integers can be of 1, 2, or 4 bytes. As rasterBuffer expects 4 byte integers we
                 * need to pad them with 0's. The order of the padding is determined by the
                 * ByteOrder of the buffer.
                 */
                if (bytespervalue == 1) {
                    cellValue = (cell.get(0) & 0xff);
                } else if (bytespervalue == 2) {
                    cellValue = cell.getShort(0);
                } else if (bytespervalue == 4) {
                    cellValue = cell.getInt(0);
                }
                /*
                 * Now write the cell value the required number of times to the raster row data
                 * buffer.
                 */
                for( int j = 0; j < count; j++ ) {
                    // // if (logger.isDebugEnabled()) logger.debug(" " +
                    // tmpint);
                    rowdata.putInt(cellValue);
                }
            }
        }
    }