Process the given record for the index.
@param rec the record from the file being indexed
@throws IOException
547108
public void processAlignment(final SAMRecord rec) throws IOException {
		// write an offset for the first record and for the g-th record thereafter (where
		// g is the granularity), to be consistent with the index method
		if (count == 0 || (count + 1) % granularity == 0) {
			SAMFileSource fileSource = rec.getFileSource();
			SAMFileSpan filePointer = fileSource.getFilePointer();
			writeVirtualOffset(getPos(filePointer));
		}
		count++;
	}
400971
def data(rec)
      @io.seek(rec.data_offset, IO::SEEK_SET)
      @io.read(rec.data_length)
    end
1271905
private void processResourceData() throws IOException
   {
      DirectoryEntry rscDir = (DirectoryEntry) m_projectDir.getEntry("TBkndRsc");
      FixFix rscFixedData = new FixFix(196, new DocumentInputStream(((DocumentEntry) rscDir.getEntry("FixFix   0"))));
      FixDeferFix rscVarData = null;
      ExtendedData rscExtData = null;

      int resources = rscFixedData.getItemCount();
      byte[] data;
      int id;
      Resource resource;
      String notes;
      ProjectCalendar calendar;

      for (int loop = 0; loop < resources; loop++)
      {
         data = rscFixedData.getByteArrayValue(loop);

         //
         // Test for a valid unique id
         //
         id = MPPUtility.getInt(data, 0);
         if (id < 1)
         {
            continue;
         }

         //
         // Blank rows can be present in MPP files. The following flag
         // appears to indicate that a row is blank, and should be
         // ignored.
         //
         if ((data[8] & 0x01) != 0)
         {
            continue;
         }

         //
         // Test to ensure this resource has not been deleted
         // This may be an array of bit flags, as per the task
         // record. I have yet to see data to support this, so
         // the simple non-zero test remains.
         //
         if (MPPUtility.getShort(data, 164) != 0)
         {
            continue;
         }

         //
         // Load the var data if we have not already done so
         //
         if (rscVarData == null)
         {
            rscVarData = new FixDeferFix(new DocumentInputStream(((DocumentEntry) rscDir.getEntry("FixDeferFix   0"))));
         }

         rscExtData = new ExtendedData(rscVarData, getOffset(data, 192));

         resource = m_file.addResource();

         resource.setAccrueAt(AccrueType.getInstance(MPPUtility.getShort(data, 20)));
         resource.setActualCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 114)) / 100));
         resource.setActualOvertimeCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 144)) / 100));
         resource.setActualWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 62)) / 100, TimeUnit.HOURS));
         resource.setAvailableFrom(MPPUtility.getTimestamp(data, 28));
         resource.setAvailableTo(MPPUtility.getTimestamp(data, 32));
         //resource.setBaseCalendar();
         resource.setBaselineCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 126)) / 100));
         resource.setBaselineWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 68)) / 100, TimeUnit.HOURS));
         resource.setCode(rscExtData.getUnicodeString(RESOURCE_CODE));
         resource.setCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 120)) / 100));
         resource.setCost(1, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST1)) / 100));
         resource.setCost(2, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST2)) / 100));
         resource.setCost(3, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST3)) / 100));
         resource.setCost(4, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST4)) / 100));
         resource.setCost(5, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST5)) / 100));
         resource.setCost(6, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST6)) / 100));
         resource.setCost(7, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST7)) / 100));
         resource.setCost(8, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST8)) / 100));
         resource.setCost(9, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST9)) / 100));
         resource.setCost(10, NumberHelper.getDouble(((double) rscExtData.getLong(RESOURCE_COST10)) / 100));
         resource.setCostPerUse(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 80)) / 100));
         resource.setDate(1, rscExtData.getTimestamp(RESOURCE_DATE1));
         resource.setDate(2, rscExtData.getTimestamp(RESOURCE_DATE2));
         resource.setDate(3, rscExtData.getTimestamp(RESOURCE_DATE3));
         resource.setDate(4, rscExtData.getTimestamp(RESOURCE_DATE4));
         resource.setDate(5, rscExtData.getTimestamp(RESOURCE_DATE5));
         resource.setDate(6, rscExtData.getTimestamp(RESOURCE_DATE6));
         resource.setDate(7, rscExtData.getTimestamp(RESOURCE_DATE7));
         resource.setDate(8, rscExtData.getTimestamp(RESOURCE_DATE8));
         resource.setDate(9, rscExtData.getTimestamp(RESOURCE_DATE9));
         resource.setDate(10, rscExtData.getTimestamp(RESOURCE_DATE10));
         resource.setDuration(1, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION1), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION1_UNITS))));
         resource.setDuration(2, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION2), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION2_UNITS))));
         resource.setDuration(3, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION3), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION3_UNITS))));
         resource.setDuration(4, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION4), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION4_UNITS))));
         resource.setDuration(5, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION5), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION5_UNITS))));
         resource.setDuration(6, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION6), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION6_UNITS))));
         resource.setDuration(7, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION7), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION7_UNITS))));
         resource.setDuration(8, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION8), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION8_UNITS))));
         resource.setDuration(9, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION9), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION9_UNITS))));
         resource.setDuration(10, MPPUtility.getDuration(rscExtData.getInt(RESOURCE_DURATION10), MPPUtility.getDurationTimeUnits(rscExtData.getShort(RESOURCE_DURATION10_UNITS))));
         resource.setEmailAddress(rscExtData.getUnicodeString(RESOURCE_EMAIL));
         resource.setFinish(1, rscExtData.getTimestamp(RESOURCE_FINISH1));
         resource.setFinish(2, rscExtData.getTimestamp(RESOURCE_FINISH2));
         resource.setFinish(3, rscExtData.getTimestamp(RESOURCE_FINISH3));
         resource.setFinish(4, rscExtData.getTimestamp(RESOURCE_FINISH4));
         resource.setFinish(5, rscExtData.getTimestamp(RESOURCE_FINISH5));
         resource.setFinish(6, rscExtData.getTimestamp(RESOURCE_FINISH6));
         resource.setFinish(7, rscExtData.getTimestamp(RESOURCE_FINISH7));
         resource.setFinish(8, rscExtData.getTimestamp(RESOURCE_FINISH8));
         resource.setFinish(9, rscExtData.getTimestamp(RESOURCE_FINISH9));
         resource.setFinish(10, rscExtData.getTimestamp(RESOURCE_FINISH10));
         resource.setGroup(rscExtData.getUnicodeString(RESOURCE_GROUP));
         resource.setID(Integer.valueOf(MPPUtility.getInt(data, 4)));
         resource.setInitials(rscVarData.getUnicodeString(getOffset(data, 160)));
         //resource.setLinkedFields(); // Calculated value
         resource.setMaxUnits(NumberHelper.getDouble(((double) MPPUtility.getInt(data, 52)) / 100));
         resource.setName(rscVarData.getUnicodeString(getOffset(data, 156)));
         resource.setNumber(1, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER1)));
         resource.setNumber(2, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER2)));
         resource.setNumber(3, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER3)));
         resource.setNumber(4, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER4)));
         resource.setNumber(5, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER5)));
         resource.setNumber(6, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER6)));
         resource.setNumber(7, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER7)));
         resource.setNumber(8, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER8)));
         resource.setNumber(9, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER9)));
         resource.setNumber(10, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER10)));
         resource.setNumber(11, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER11)));
         resource.setNumber(12, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER12)));
         resource.setNumber(13, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER13)));
         resource.setNumber(14, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER14)));
         resource.setNumber(15, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER15)));
         resource.setNumber(16, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER16)));
         resource.setNumber(17, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER17)));
         resource.setNumber(18, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER18)));
         resource.setNumber(19, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER19)));
         resource.setNumber(20, NumberHelper.getDouble(rscExtData.getDouble(RESOURCE_NUMBER20)));
         //resource.setObjects(); // Calculated value
         //resource.setOverallocated(); // Calculated value
         resource.setOvertimeCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 138)) / 100));
         resource.setOvertimeRate(new Rate(MPPUtility.getDouble(data, 44), TimeUnit.HOURS));
         resource.setOvertimeWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 74)) / 100, TimeUnit.HOURS));
         resource.setPeakUnits(NumberHelper.getDouble(((double) MPPUtility.getInt(data, 110)) / 100));
         //resource.setPercentageWorkComplete(); // Calculated value
         resource.setRegularWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 92)) / 100, TimeUnit.HOURS));
         resource.setRemainingCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 132)) / 100));
         resource.setRemainingOvertimeCost(NumberHelper.getDouble(((double) MPPUtility.getLong6(data, 150)) / 100));
         resource.setRemainingWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 86)) / 100, TimeUnit.HOURS));
         resource.setStandardRate(new Rate(MPPUtility.getDouble(data, 36), TimeUnit.HOURS));
         resource.setStart(1, rscExtData.getTimestamp(RESOURCE_START1));
         resource.setStart(2, rscExtData.getTimestamp(RESOURCE_START2));
         resource.setStart(3, rscExtData.getTimestamp(RESOURCE_START3));
         resource.setStart(4, rscExtData.getTimestamp(RESOURCE_START4));
         resource.setStart(5, rscExtData.getTimestamp(RESOURCE_START5));
         resource.setStart(6, rscExtData.getTimestamp(RESOURCE_START6));
         resource.setStart(7, rscExtData.getTimestamp(RESOURCE_START7));
         resource.setStart(8, rscExtData.getTimestamp(RESOURCE_START8));
         resource.setStart(9, rscExtData.getTimestamp(RESOURCE_START9));
         resource.setStart(10, rscExtData.getTimestamp(RESOURCE_START10));
         resource.setText(1, rscExtData.getUnicodeString(RESOURCE_TEXT1));
         resource.setText(2, rscExtData.getUnicodeString(RESOURCE_TEXT2));
         resource.setText(3, rscExtData.getUnicodeString(RESOURCE_TEXT3));
         resource.setText(4, rscExtData.getUnicodeString(RESOURCE_TEXT4));
         resource.setText(5, rscExtData.getUnicodeString(RESOURCE_TEXT5));
         resource.setText(6, rscExtData.getUnicodeString(RESOURCE_TEXT6));
         resource.setText(7, rscExtData.getUnicodeString(RESOURCE_TEXT7));
         resource.setText(8, rscExtData.getUnicodeString(RESOURCE_TEXT8));
         resource.setText(9, rscExtData.getUnicodeString(RESOURCE_TEXT9));
         resource.setText(10, rscExtData.getUnicodeString(RESOURCE_TEXT10));
         resource.setText(11, rscExtData.getUnicodeString(RESOURCE_TEXT11));
         resource.setText(12, rscExtData.getUnicodeString(RESOURCE_TEXT12));
         resource.setText(13, rscExtData.getUnicodeString(RESOURCE_TEXT13));
         resource.setText(14, rscExtData.getUnicodeString(RESOURCE_TEXT14));
         resource.setText(15, rscExtData.getUnicodeString(RESOURCE_TEXT15));
         resource.setText(16, rscExtData.getUnicodeString(RESOURCE_TEXT16));
         resource.setText(17, rscExtData.getUnicodeString(RESOURCE_TEXT17));
         resource.setText(18, rscExtData.getUnicodeString(RESOURCE_TEXT18));
         resource.setText(19, rscExtData.getUnicodeString(RESOURCE_TEXT19));
         resource.setText(20, rscExtData.getUnicodeString(RESOURCE_TEXT20));
         resource.setText(21, rscExtData.getUnicodeString(RESOURCE_TEXT21));
         resource.setText(22, rscExtData.getUnicodeString(RESOURCE_TEXT22));
         resource.setText(23, rscExtData.getUnicodeString(RESOURCE_TEXT23));
         resource.setText(24, rscExtData.getUnicodeString(RESOURCE_TEXT24));
         resource.setText(25, rscExtData.getUnicodeString(RESOURCE_TEXT25));
         resource.setText(26, rscExtData.getUnicodeString(RESOURCE_TEXT26));
         resource.setText(27, rscExtData.getUnicodeString(RESOURCE_TEXT27));
         resource.setText(28, rscExtData.getUnicodeString(RESOURCE_TEXT28));
         resource.setText(29, rscExtData.getUnicodeString(RESOURCE_TEXT29));
         resource.setText(30, rscExtData.getUnicodeString(RESOURCE_TEXT30));
         resource.setUniqueID(Integer.valueOf(id));
         resource.setWork(MPPUtility.getDuration(((double) MPPUtility.getLong6(data, 56)) / 100, TimeUnit.HOURS));

         //
         // Attach the resource calendar
         //
         calendar = m_calendarMap.get(Integer.valueOf(MPPUtility.getInt(data, 24)));
         resource.setResourceCalendar(calendar);

         //
         // Retrieve the resource notes.
         //
         notes = rscExtData.getString(RESOURCE_NOTES);
         if (notes != null)
         {
            if (m_reader.getPreserveNoteFormatting() == false)
            {
               notes = RtfHelper.strip(notes);
            }

            resource.setNotes(notes);
         }

         m_eventManager.fireResourceReadEvent(resource);
      }
   }
883737
@Override
	protected void processRecord(Tuple2<Boolean, Row> change) {
		synchronized (resultLock) {
			// insert
			if (change.f0) {
				processInsert(change.f1);
			}
			// delete
			else {
				processDelete(change.f1);
			}
		}
	}
715317
public void processFile(File blogFile) throws IOException {
    BufferedReader br = new BufferedReader(new FileReader(blogFile));
    String line = null;
    String date = null;
    String id = null;
    StringBuilder content = new StringBuilder();
    boolean needMoreContent = false;
    while ((line = br.readLine()) != null) {
      if (line.contains("<id>")) {
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        id = line.substring(startIndex, endIndex);
      } else if (line.contains("<content>")) {
        // Extract the start of a content node.  If the previous content,
        // updated pair was incomplete, i.e. updated had no value, this will
        // overwrite the previous content value.
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        content = new StringBuilder();
        if (endIndex > startIndex)
          content.append(line.substring(startIndex, endIndex));
        else {
          content.append(line.substring(startIndex));
          needMoreContent = true;
        }
      } else if (needMoreContent) {
        // The content node might span several lines, so consider all lines read
        // until the next close bracket to be part of the current content.
        int endIndex = (line.contains("</content>")) ? line.lastIndexOf("<") : -1;
        if (endIndex > 0) {
          content.append(line.substring(0, endIndex));
          needMoreContent = false;
        } else
          content.append(line);
      } else if (line.contains("<updated>")) {
        // The updated timestamp only spans one line.
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        date = line.substring(startIndex, endIndex);
        if (date.equals(""))
          date = null;
      } else if (content != null && date != null) {
        // Cleand and print out the content and date.
        long dateTime = Timestamp.valueOf(date).getTime();
        if (dateTime < beginTime || dateTime > endTime) {
          needMoreContent = false;
          date = null;
          continue;
        }
        String cleanedContent = processor.process(content.toString());
        if (!cleanedContent.equals("")) {
          synchronized (pw) {
            pw.format("%d %s\n", dateTime, cleanedContent);
            pw.flush();
          }
        }
        LOGGER.info(String.format("Processed blog %s with timestamp %d",
                                  id, dateTime));
        needMoreContent = false;
        date = null;
      }
    }
    br.close();
  }
368720
public void fieldsToData(Rec record) throws DBException
    {
        try   {
            int iFieldTypes = BaseBuffer.PHYSICAL_FIELDS;
            if (!((Record)record).isAllSelected())
                iFieldTypes = BaseBuffer.DATA_FIELDS;   //SELECTED_FIELDS; (Selected and physical)
            m_dataSource = new VectorBuffer(null, iFieldTypes);   //x new StringBuff();
            ((BaseBuffer)m_dataSource).fieldsToBuffer((FieldList)record);
        } catch (Exception ex) {
            throw DatabaseException.toDatabaseException(ex);
        }
    }
113040
BlockCrcInfoWritable getNextRecord() throws IOException {
    // By calling getBucketIdForNextRecord(), we make sure the next field
    // to read is the next record (if there is any record left in the file)
    // Also, by checking the return value, we know whether we've finished
    // the file.
    if (moveToNextRecordAndGetItsBucketId() == -1) {
      return null;
    }

    BlockCrcInfoWritable crcInfo = new BlockCrcInfoWritable();
    crcInfo.readFields(in);
    numRecordsReadInBucket++;
    return crcInfo;
  }
306846
public void skipRecord() throws IOException {
        if (this.debug) {
            System.err.println("SkipRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx);
        }

        if (this.inStream == null) {
            throw new IOException("reading (via skip) from an output buffer");
        }

        if (this.currRecIdx >= this.recsPerBlock) {
            if (!this.readBlock()) {
                return; // UNDONE
            }
        }

        this.currRecIdx++;
    }
5782
private void processFile(InputStream is) throws MPXJException
   {
      int line = 1;

      try
      {
         //
         // Test the header and extract the separator. If this is successful,
         // we reset the stream back as far as we can. The design of the
         // BufferedInputStream class means that we can't get back to character
         // zero, so the first record we will read will get "RMHDR" rather than
         // "ERMHDR" in the first field position.
         //
         BufferedInputStream bis = new BufferedInputStream(is);
         byte[] data = new byte[6];
         data[0] = (byte) bis.read();
         bis.mark(1024);
         bis.read(data, 1, 5);

         if (!new String(data).equals("ERMHDR"))
         {
            throw new MPXJException(MPXJException.INVALID_FILE);
         }

         bis.reset();

         InputStreamReader reader = new InputStreamReader(bis, getCharset());
         Tokenizer tk = new ReaderTokenizer(reader);
         tk.setDelimiter('\t');
         List<String> record = new ArrayList<String>();

         while (tk.getType() != Tokenizer.TT_EOF)
         {
            readRecord(tk, record);
            if (!record.isEmpty())
            {
               if (processRecord(record))
               {
                  break;
               }
            }
            ++line;
         }
      }

      catch (Exception ex)
      {
         throw new MPXJException(MPXJException.READ_ERROR + " (failed at line " + line + ")", ex);
      }
   }
561217
protected synchronized void makeIndex() throws Exception {
        File indexFile = getIndexFile(filename);
        if (indexFile.exists()) try {
            loadIndex(indexFile);
            setIndexCreated(true);
            return;
        } catch (Exception x) {
            logger.warn(x.getMessage());
        }
        indexCreated = false;
        long now = System.currentTimeMillis();
        int recordLength = 1000;
        int maxRecords = 1;
        int maxRecordLength = 0;
        maxRecords = (int) raFile.length() / recordLength;
        if (maxRecords == 0) maxRecords = 1;
        index = new long[maxRecords][3];

        String s = null;
        long start = 0;
        long end = 0;
        raFile.seek(0);
        records = 0;
        recordLength = 0;
        while ((s = raFile.readLine()) != null) {
            if (start == -1) start = raFile.getFilePointer();
            if (isRecordEnd(s)) {
                //fireFrameRead();
                if (records >= maxRecords) {
                    index = resize(index,
                            records
                                    + (int) (records + (raFile.length() - records * raFile.getFilePointer())
                                            / recordLength));
                }
                end += 4;
                index[records][0] = start;
                index[records][1] = end - start;
                index[records][2] = -1;
                if (maxRecordLength < index[records][1]) maxRecordLength = (int) index[records][1];
                records++;
                recordLength += end - start;

                start = raFile.getFilePointer();
            } else {
                end = raFile.getFilePointer();
            }

        }
        b = new byte[maxRecordLength];
        //fireFrameRead();
        logger.info("Index created in " + (System.currentTimeMillis() - now) + " ms.");
        try {
            saveIndex(indexFile);
        } catch (Exception x) {
            logger.error(x);
        }
    }
1124425
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // prefetch the first windowSize words 
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i)
            nextWords.offer(documentTokens.next());
        
        while (!nextWords.isEmpty()) {
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);
            
            if (calculateSemantics) {
                IntegerVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    add(focusMeaning, iv);
                }
            
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }

                    add(focusMeaning, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);
            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
1336923
@Override
	public void process(ResponseBuilder rb) throws IOException {
		// System.out
		// .println(System.nanoTime() + " - " + Thread.currentThread().getId()
		// + " - " + rb.req.getParams().getBool(ShardParams.IS_SHARD, false)
		// + " PROCESS " + rb.stage + " " + rb.req.getParamString());
		MtasSolrStatus solrStatus = Objects
				.requireNonNull((MtasSolrStatus) rb.req.getContext().get(MtasSolrStatus.class), "couldn't find status");
		solrStatus.setStage(rb.stage);
		try {
			if (rb.req.getParams().getBool(PARAM_MTAS, false)) {
				try {
					ComponentFields mtasFields = getMtasFields(rb);
					if (mtasFields != null) {
						DocSet docSet = rb.getResults().docSet;
						DocList docList = rb.getResults().docList;
						if (mtasFields.doStats || mtasFields.doDocument || mtasFields.doKwic || mtasFields.doList
								|| mtasFields.doGroup || mtasFields.doFacet || mtasFields.doCollection
								|| mtasFields.doTermVector || mtasFields.doPrefix || mtasFields.doStatus
								|| mtasFields.doVersion) {
							SolrIndexSearcher searcher = rb.req.getSearcher();
							ArrayList<Integer> docSetList = null;
							ArrayList<Integer> docListList = null;
							// initialise docSetList
							if (docSet != null) {
								docSetList = new ArrayList<>();
								Iterator<Integer> docSetIterator = docSet.iterator();
								while (docSetIterator.hasNext()) {
									docSetList.add(docSetIterator.next());
								}
								Collections.sort(docSetList);
							}
							// initialise docListList
							if (docList != null) {
								docListList = new ArrayList<>();
								Iterator<Integer> docListIterator = docList.iterator();
								while (docListIterator.hasNext()) {
									docListList.add(docListIterator.next());
								}
								Collections.sort(docListList);
							}
							solrStatus.status().addSubs(mtasFields.list.keySet());
							for (String field : mtasFields.list.keySet()) {
								try {
									CodecUtil.collectField(field, searcher, searcher.getRawReader(), docListList,
											docSetList, mtasFields.list.get(field), solrStatus.status());
								} catch (IllegalAccessException | IllegalArgumentException
										| InvocationTargetException e) {
									log.error(e);
									throw new IOException(e);
								}
							}
							for (ComponentCollection collection : mtasFields.collection) {
								CodecUtil.collectCollection(searcher.getRawReader(), docSetList, collection);
							}
							NamedList<Object> mtasResponse = new SimpleOrderedMap<>();
							if (mtasFields.doVersion) {
								SimpleOrderedMap<Object> versionResponse = searchVersion.create(mtasFields.version,
										false);
								mtasResponse.add(MtasSolrComponentVersion.NAME, versionResponse);
							}
							if (mtasFields.doStatus) {
								// add to response
								SimpleOrderedMap<Object> statusResponse = searchStatus.create(mtasFields.status, false);
								if (statusResponse != null) {
									mtasResponse.add(MtasSolrComponentStatus.NAME,
											searchStatus.create(mtasFields.status, false));
								}
							}
							if (mtasFields.doDocument) {
								ArrayList<NamedList<?>> mtasDocumentResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentDocument document : mtasFields.list.get(field).documentList) {
										mtasDocumentResponses.add(searchDocument.create(document, false));
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentDocument.NAME, mtasDocumentResponses);
							}
							if (mtasFields.doKwic) {
								ArrayList<NamedList<?>> mtasKwicResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentKwic kwic : mtasFields.list.get(field).kwicList) {
										mtasKwicResponses.add(searchKwic.create(kwic, false));
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentKwic.NAME, mtasKwicResponses);
							}
							if (mtasFields.doFacet) {
								ArrayList<NamedList<?>> mtasFacetResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentFacet facet : mtasFields.list.get(field).facetList) {
										if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
											mtasFacetResponses.add(searchFacet.create(facet, true));
										} else {
											mtasFacetResponses.add(searchFacet.create(facet, false));
										}
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentFacet.NAME, mtasFacetResponses);
							}
							if (mtasFields.doCollection) {
								ArrayList<NamedList<?>> mtasCollectionResponses = new ArrayList<>();
								for (ComponentCollection collection : mtasFields.collection) {
									if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
										mtasCollectionResponses.add(searchCollection.create(collection, true));
									} else {
										mtasCollectionResponses.add(searchCollection.create(collection, false));
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentCollection.NAME, mtasCollectionResponses);
							}
							if (mtasFields.doList) {
								ArrayList<NamedList<?>> mtasListResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentList list : mtasFields.list.get(field).listList) {
										mtasListResponses.add(searchList.create(list, false));
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentList.NAME, mtasListResponses);
							}
							if (mtasFields.doGroup) {
								ArrayList<NamedList<?>> mtasGroupResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentGroup group : mtasFields.list.get(field).groupList) {
										if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
											mtasGroupResponses.add(searchGroup.create(group, true));
										} else {
											mtasGroupResponses.add(searchGroup.create(group, false));
										}
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentGroup.NAME, mtasGroupResponses);
							}
							if (mtasFields.doTermVector) {
								ArrayList<NamedList<?>> mtasTermVectorResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									for (ComponentTermVector termVector : mtasFields.list.get(field).termVectorList) {
										if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
											mtasTermVectorResponses.add(searchTermvector.create(termVector, true));
										} else {
											mtasTermVectorResponses.add(searchTermvector.create(termVector, false));
										}
									}
								}
								// add to response
								mtasResponse.add(MtasSolrComponentTermvector.NAME, mtasTermVectorResponses);
							}
							if (mtasFields.doPrefix) {
								ArrayList<NamedList<?>> mtasPrefixResponses = new ArrayList<>();
								for (String field : mtasFields.list.keySet()) {
									if (mtasFields.list.get(field).prefix != null) {
										if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
											mtasPrefixResponses
													.add(searchPrefix.create(mtasFields.list.get(field).prefix, true));
										} else {
											mtasPrefixResponses
													.add(searchPrefix.create(mtasFields.list.get(field).prefix, false));
										}
									}
								}
								mtasResponse.add(MtasSolrComponentPrefix.NAME, mtasPrefixResponses);
							}
							if (mtasFields.doStats) {
								NamedList<Object> mtasStatsResponse = new SimpleOrderedMap<>();
								if (mtasFields.doStatsPositions || mtasFields.doStatsTokens
										|| mtasFields.doStatsSpans) {
									if (mtasFields.doStatsTokens) {
										ArrayList<Object> mtasStatsTokensResponses = new ArrayList<>();
										for (String field : mtasFields.list.keySet()) {
											for (ComponentToken token : mtasFields.list.get(field).statsTokenList) {
												if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
													mtasStatsTokensResponses.add(searchStats.create(token, true));
												} else {
													mtasStatsTokensResponses.add(searchStats.create(token, false));
												}
											}
										}
										mtasStatsResponse.add(MtasSolrComponentStats.NAME_TOKENS,
												mtasStatsTokensResponses);
									}
									if (mtasFields.doStatsPositions) {
										ArrayList<Object> mtasStatsPositionsResponses = new ArrayList<>();
										for (String field : mtasFields.list.keySet()) {
											for (ComponentPosition position : mtasFields.list
													.get(field).statsPositionList) {
												if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
													mtasStatsPositionsResponses.add(searchStats.create(position, true));
												} else {
													mtasStatsPositionsResponses
															.add(searchStats.create(position, false));
												}
											}
										}
										mtasStatsResponse.add(MtasSolrComponentStats.NAME_POSITIONS,
												mtasStatsPositionsResponses);
									}
									if (mtasFields.doStatsSpans) {
										ArrayList<Object> mtasStatsSpansResponses = new ArrayList<>();
										for (String field : mtasFields.list.keySet()) {
											for (ComponentSpan span : mtasFields.list.get(field).statsSpanList) {
												if (rb.req.getParams().getBool(ShardParams.IS_SHARD, false)) {
													mtasStatsSpansResponses.add(searchStats.create(span, true));
												} else {
													mtasStatsSpansResponses.add(searchStats.create(span, false));
												}
											}
										}
										mtasStatsResponse.add(MtasSolrComponentStats.NAME_SPANS,
												mtasStatsSpansResponses);
									}
									// add to response
									mtasResponse.add(MtasSolrComponentStats.NAME, mtasStatsResponse);
								}
							}
							// add to response
							if (mtasResponse.size() > 0) {
								rb.rsp.add(NAME, mtasResponse);
							}
						}
					}
				} catch (IOException e) {
					errorStatus(solrStatus, e);
				}
			}
			if (!solrStatus.error()) {
				// always set status segments
				if (solrStatus.status().numberSegmentsTotal == null) {
					solrStatus.status().numberSegmentsTotal = rb.req.getSearcher().getRawReader().leaves().size();
					solrStatus.status().numberSegmentsFinished = solrStatus.status().numberSegmentsTotal;
				}
				// always try to set number of documents
				if (solrStatus.status().numberDocumentsTotal == null) {
					SolrIndexSearcher searcher;
					if ((searcher = rb.req.getSearcher()) != null) {
						solrStatus.status().numberDocumentsTotal = (long) searcher.numDocs();
						if (rb.getResults().docList != null) {
							solrStatus.status().numberDocumentsFinished = rb.getResults().docList.matches();
							solrStatus.status().numberDocumentsFound = rb.getResults().docList.matches();
						} else if (rb.getResults().docSet != null) {
							solrStatus.status().numberDocumentsFinished = (long) rb.getResults().docSet.size();
							solrStatus.status().numberDocumentsFound = (long) rb.getResults().docSet.size();
						}
					}
				}
			}
		} catch (ExitableDirectoryReader.ExitingReaderException e) {
			solrStatus.setError(e.getMessage());
		} finally {
			checkStatus(solrStatus);
			finishStatus(solrStatus);
		}
	}
862611
public void processEvent(LogRecord record) {
		byte[] bytes;

		SerializationObject serializationObject = pool.getSerializationObject();
		try {
			bytes = serializationObject.serialize(record);
		} finally {
			pool.returnSerializationObject(serializationObject);
		}

		synchronized(this) {
			if (traceWriter != null && record.getLevel().intValue() < traceThreshold) {
				traceWriter.logRecord(record.getMillis(), bytes);
			} else if (logWriter != null) {
				logWriter.logRecord(record.getMillis(), bytes);
			}
		}
	}
282912
private void handleReadBuffer() throws IOException {
        if (readBuffer.remaining() == 0) {
            readBuffer.clear();
            actualFile.read(actualFileOffset, readBuffer);
            actualFileOffset += readBuffer.limit();
            readBuffer.position(0);
        }
    }
1267168
public int getRecordCount(Rec record)
    {
        int iRecordCount = 1;
        if ((this.getNodeType() == MessageRecordDesc.NON_UNIQUE_NODE)
            && (!this.isSingleDetail(record)))
                iRecordCount = Integer.MAX_VALUE;   // Until EOF
        return iRecordCount;
    }
148290
protected void setReadingPosition(int recordIndex, int byteIndex) throws IOException {
		if (this.seekEnabled) {
			this.nextExpectedRecordIndex = recordIndex;
			this.buffer.position(byteIndex);
		} else {
			throw new SeekOperationDisabledException();
		}
	}
108568
private void processRecords(Hadoop2RecordType recType, JSONObject eventDetails)
      throws JSONException {

    switch (recType) {
    case JobFinished:
      // this setting is needed since the job history file is missing
      // the jobStatus field in the JOB_FINISHED event
      this.jobStatus = JOB_STATUS_SUCCEEDED;
    case JobInfoChange:
    case JobInited:
    case JobPriorityChange:
    case JobStatusChanged:
    case JobQueueChange:
    case JobSubmitted:
    case JobUnsuccessfulCompletion:
      Put pJob = new Put(this.jobKeyBytes);
      iterateAndPreparePuts(eventDetails, pJob, recType);
      this.jobPuts.add(pJob);
      break;

    case AMStarted:
      byte[] amAttemptIdKeyBytes =
          getAMKey(AM_ATTEMPT_PREFIX, eventDetails.getString(APPLICATION_ATTEMPTID));
      // generate a new put per AM Attempt
      Put pAM = new Put(amAttemptIdKeyBytes);
      pAM.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pAM, recType);
      taskPuts.add(pAM);
      break;

    case MapAttemptFinished:
      byte[] taskMAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pMTaskAttempt = new Put(taskMAttemptIdKeyBytes);
      pMTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.MapAttempt.toString()));
      iterateAndPreparePuts(eventDetails, pMTaskAttempt, recType);
      this.taskPuts.add(pMTaskAttempt);
      break;

    case ReduceAttemptFinished:
      byte[] taskRAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pRTaskAttempt = new Put(taskRAttemptIdKeyBytes);
      pRTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.ReduceAttempt.toString()));
      iterateAndPreparePuts(eventDetails, pRTaskAttempt, recType);
      this.taskPuts.add(pRTaskAttempt);
      break;

    case TaskAttemptFinished:
    case TaskAttemptStarted:
    case TaskAttemptUnsuccessfulCompletion:
      byte[] taskAttemptIdKeyBytes =
          getTaskKey(TASK_ATTEMPT_PREFIX, this.jobNumber, eventDetails.getString(ATTEMPTID));
      Put pTaskAttempt = new Put(taskAttemptIdKeyBytes);
      pTaskAttempt.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pTaskAttempt, recType);
      taskPuts.add(pTaskAttempt);
      break;

    case TaskFailed:
    case TaskStarted:
    case TaskUpdated:
    case TaskFinished:
      byte[] taskIdKeyBytes =
          getTaskKey(TASK_PREFIX, this.jobNumber, eventDetails.getString(TASKID));
      Put pTask = new Put(taskIdKeyBytes);
      pTask.addColumn(Constants.INFO_FAM_BYTES, Constants.RECORD_TYPE_COL_BYTES,
        Bytes.toBytes(RecordTypes.Task.toString()));
      iterateAndPreparePuts(eventDetails, pTask, recType);
      taskPuts.add(pTask);
      break;
    default:
      LOG.error("Check if recType was modified and has new members?");
      throw new ProcessingException("Check if recType was modified and has new members? " + recType);
    }
  }