Read, parse and optionally tail the log file, converting entries into logging events.

A runtimeException is thrown if the logFormat pattern is malformed.

@param bufferedReader
@throws IOException
335484
protected void process(BufferedReader bufferedReader) throws IOException {
        Matcher eventMatcher;
        Matcher exceptionMatcher;
        String line;
        while ((line = bufferedReader.readLine()) != null) {
            //skip empty line entries
            eventMatcher = regexpPattern.matcher(line);
            if (line.trim().equals("")) {continue;}
            exceptionMatcher = exceptionPattern.matcher(line);
            if (eventMatcher.matches()) {
                //build an event from the previous match (held in current map)
                LoggingEvent event = buildEvent();
                if (event != null) {
                    if (passesExpression(event)) {
                        doPost(event);
                    }
                }
                currentMap.putAll(processEvent(eventMatcher.toMatchResult()));
            } else if (exceptionMatcher.matches()) {
                //an exception line
                additionalLines.add(line);
            } else {
                //neither...either post an event with the line or append as additional lines
                //if this was a logging event with multiple lines, each line will show up as its own event instead of being
                //appended as multiple lines on the same event..
                //choice is to have each non-matching line show up as its own line, or append them all to a previous event
                if (appendNonMatches) {
                    //hold on to the previous time, so we can do our best to preserve time-based ordering if the event is a non-match
                    String lastTime = (String)currentMap.get(TIMESTAMP);
                    //build an event from the previous match (held in current map)
                    if (currentMap.size() > 0) {
                        LoggingEvent event = buildEvent();
                        if (event != null) {
                            if (passesExpression(event)) {
                              doPost(event);
                            }
                        }
                    }
                    if (lastTime != null) {
                        currentMap.put(TIMESTAMP, lastTime);
                    }
                    currentMap.put(MESSAGE, line);
                } else {
                    additionalLines.add(line);
                }
            }
        }

        //process last event if one exists
        LoggingEvent event = buildEvent();
        if (event != null) {
            if (passesExpression(event)) {
                doPost(event);
            }
        }
    }
105377
private List<String> processLine(String line,
			BufferedReader buf,
			int fieldLength)
					throws IOException{

		//System.out.println("XX processLine " + fieldLength + " " + line);
		// go through the line and process each character
		List<String> lineData = new ArrayList<String>();

		boolean inString = false;

		StringBuilder bigWord = null;

		while ( true ){

			if ( line.startsWith(STRING_LIMIT)){
				if (! inString){

					inString = true;
					if ( line.length() > 1)
						bigWord = new StringBuilder(line.substring(1));
					else
						bigWord = new StringBuilder("");


				} else {
					// the end of a word
					lineData.add(bigWord.toString());
					bigWord = null;
					inString = false;

				}
			} else {
				if ( inString )
					bigWord.append(line);
				else {

					List<String> dat = processSingleLine(line);

					for (String d : dat){
						lineData.add(d);
					}
				}
			}

			//System.out.println("in process line : " + lineData.size() + " " + fieldLength);

			if ( lineData.size() > fieldLength){

				logger.warn("wrong data length ("+lineData.size()+
						") should be ("+fieldLength+") at line " + line + " got lineData: " + lineData);
				return lineData;
			}

			if ( lineData.size() == fieldLength)
				return lineData;


			line = buf.readLine();
			if ( line == null)
				break;
		}
		return lineData;

	}
271209
private void readAsync() throws IOException {
    stateChangeLock.lock();
    final byte[] arr = readAheadBuffer.array();
    try {
      if (endOfStream || readInProgress) {
        return;
      }
      checkReadException();
      readAheadBuffer.position(0);
      readAheadBuffer.flip();
      readInProgress = true;
    } finally {
      stateChangeLock.unlock();
    }
    executorService.execute(() -> {
      stateChangeLock.lock();
      try {
        if (isClosed) {
          readInProgress = false;
          return;
        }
        // Flip this so that the close method will not close the underlying input stream when we
        // are reading.
        isReading = true;
      } finally {
        stateChangeLock.unlock();
      }

      // Please note that it is safe to release the lock and read into the read ahead buffer
      // because either of following two conditions will hold - 1. The active buffer has
      // data available to read so the reader will not read from the read ahead buffer.
      // 2. This is the first time read is called or the active buffer is exhausted,
      // in that case the reader waits for this async read to complete.
      // So there is no race condition in both the situations.
      int read = 0;
      int off = 0, len = arr.length;
      Throwable exception = null;
      try {
        // try to fill the read ahead buffer.
        // if a reader is waiting, possibly return early.
        do {
          read = underlyingInputStream.read(arr, off, len);
          if (read <= 0) break;
          off += read;
          len -= read;
        } while (len > 0 && !isWaiting.get());
      } catch (Throwable ex) {
        exception = ex;
        if (ex instanceof Error) {
          // `readException` may not be reported to the user. Rethrow Error to make sure at least
          // The user can see Error in UncaughtExceptionHandler.
          throw (Error) ex;
        }
      } finally {
        stateChangeLock.lock();
        readAheadBuffer.limit(off);
        if (read < 0 || (exception instanceof EOFException)) {
          endOfStream = true;
        } else if (exception != null) {
          readAborted = true;
          readException = exception;
        }
        readInProgress = false;
        signalAsyncReadComplete();
        stateChangeLock.unlock();
        closeUnderlyingInputStreamIfNecessary();
      }
    });
  }
185231
public void readDump() throws IOException {
		try {
			SAXParserFactory factory = SAXParserFactory.newInstance();
			factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, false);
			SAXParser parser = factory.newSAXParser();
	
			parser.parse(input, this);
		} catch (ParserConfigurationException e) {
			throw (IOException)new IOException(e.getMessage()).initCause(e);
		} catch (SAXException e) {
			throw (IOException)new IOException(e.getMessage()).initCause(e);
		}
		writer.close();
	}
1119593
private void readTrailer(final GZIPArchive.ReadEntry entry) throws IOException {

		// CRC32

		entry.crc32 = readLEInt(input);
		if (LOGGER.isTraceEnabled()) LOGGER.trace("CRC read from stream {}", Integer.valueOf(entry.crc32));

		// ISIZE

		final int iSize = readLEInt(input);
		if (entry.uncompressedSkipLength != iSize) throw new GZIPArchive.FormatException("Length mismatch between (warc) extra gzip fields uncompressed-skip-length (" + entry.uncompressedSkipLength + ") and ISIZE (" + iSize + ")");

	}
941004
private static BufferedReader getTaskLogReader(final URL taskLogUrl)
      throws IOException {
    FutureTask<BufferedReader> task = new FutureTask<BufferedReader>(
        new Callable<BufferedReader>() {
          @Override
          public BufferedReader call() throws IOException {
            URLConnection connection = taskLogUrl.openConnection();
            BufferedReader input = new BufferedReader(
                new InputStreamReader(connection.getInputStream()));
            return input;
          }
        });
    Thread thread = new Thread(task);
    thread.setDaemon(true);
    thread.start();
    BufferedReader result = null;
    try {
      // Fails if we cannot open the input stream for 5 minutes.
      // This prevents JobClient from hanging
      result = task.get(5, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
      throw new IOException(e);
    } catch (TimeoutException e) {
      throw new IOException(e);
    } catch (ExecutionException e) {
      throw new IOException(e);
    }
    if (result == null) {
      throw new IOException("Failed to open input stream for " + taskLogUrl);
    }
    return result;
  }
305379
public void rewind() throws IOException {
    if (!rewindable) {
      throw LOG.unableToRewindReader();
    }

    wrappedReader.unread(buffer, 0, pos);
    pos = 0;
  }
221919
@Override
    public int read(byte[] output) throws IOException {
        
        //Return -1 here because if we are closed or finished, we're at the end of the stream
        if(isClosed || isFinished()){
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
                Tr.debug(tc, "Nothing to read, stream is closed : " + isClosed + ", or finished : " + isFinished());
            return -1;
        }
        
        if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) {
            Tr.entry(tc, "read" , "SRTInputStream31.read(byte[])");
        }
        if(output == null){
            if (TraceComponent.isAnyTracingEnabled() && tc.isErrorEnabled())
                Tr.error(tc, "read.write.bytearray.null");
            throw new NullPointerException(Tr.formatMessage(tc, "read.write.bytearray.null"));
        }
        
        int returnSize = 0;
        
        if(this.request.isAsyncStarted() && (this.getReadListener()!= null)){
            synchronized(this){
                
                //Check if isReady had returned false. If it did then an IllegalArgumentException will be thrown if a listener is set
                isReadyFalseCheck();
                returnSize = super.read(output);
            }
        }else{
            returnSize = super.read(output);
        }
        
        if (TraceComponent.isAnyTracingEnabled() && tc.isEntryEnabled()) {
            Tr.exit(tc, "read" , "SRTInputStream31.read(byte[]) : " + returnSize);
        }
        
        return returnSize;
    }
715317
public void processFile(File blogFile) throws IOException {
    BufferedReader br = new BufferedReader(new FileReader(blogFile));
    String line = null;
    String date = null;
    String id = null;
    StringBuilder content = new StringBuilder();
    boolean needMoreContent = false;
    while ((line = br.readLine()) != null) {
      if (line.contains("<id>")) {
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        id = line.substring(startIndex, endIndex);
      } else if (line.contains("<content>")) {
        // Extract the start of a content node.  If the previous content,
        // updated pair was incomplete, i.e. updated had no value, this will
        // overwrite the previous content value.
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        content = new StringBuilder();
        if (endIndex > startIndex)
          content.append(line.substring(startIndex, endIndex));
        else {
          content.append(line.substring(startIndex));
          needMoreContent = true;
        }
      } else if (needMoreContent) {
        // The content node might span several lines, so consider all lines read
        // until the next close bracket to be part of the current content.
        int endIndex = (line.contains("</content>")) ? line.lastIndexOf("<") : -1;
        if (endIndex > 0) {
          content.append(line.substring(0, endIndex));
          needMoreContent = false;
        } else
          content.append(line);
      } else if (line.contains("<updated>")) {
        // The updated timestamp only spans one line.
        int startIndex = line.indexOf(">")+1;
        int endIndex = line.lastIndexOf("<");
        date = line.substring(startIndex, endIndex);
        if (date.equals(""))
          date = null;
      } else if (content != null && date != null) {
        // Cleand and print out the content and date.
        long dateTime = Timestamp.valueOf(date).getTime();
        if (dateTime < beginTime || dateTime > endTime) {
          needMoreContent = false;
          date = null;
          continue;
        }
        String cleanedContent = processor.process(content.toString());
        if (!cleanedContent.equals("")) {
          synchronized (pw) {
            pw.format("%d %s\n", dateTime, cleanedContent);
            pw.flush();
          }
        }
        LOGGER.info(String.format("Processed blog %s with timestamp %d",
                                  id, dateTime));
        needMoreContent = false;
        date = null;
      }
    }
    br.close();
  }
1314185
@Override
    public int read() throws IOException {
        if (metrics != null) {
            metrics.startEvent(Field.HttpSocketReadTime);
        }
        try {
            return in.read();
        } finally {
            if (metrics != null) {
                metrics.endEvent(Field.HttpSocketReadTime);
            }
        }
    }
671486
void parseOneFile(final Path inputPath, final File outputFolder) throws IOException {
    try (final TFile.Reader.Scanner scanner = this.getScanner(inputPath)) {
      while (!scanner.atEnd()) {
        new LogFileEntry(scanner.entry()).write(outputFolder);
        scanner.advance();
      }
    }
  }
802521
public static LogView parse(File file) throws IOException {
                LogView view = (LogView) xstream.fromXML(file);
                view.reinitialize();
                return view;
        }
980266
public void parse(InputStream is) throws IOException {
		
		BufferedReader br = new BufferedReader(new InputStreamReader(
				(InputStream) is,ByteOp.UTF8));
        String read;
        boolean allowRuleFound = false;
        // true if curr or last line read was a User-agent line 
        boolean currLineUA = false; 
        boolean lastLineUA = false;
        ArrayList<String> current = null;
        while (br != null) {
            lastLineUA = currLineUA;
            do {
                read = br.readLine();
                // Skip comments & blanks
            } while ((read != null) && ((read = read.trim()).startsWith("#") ||
                read.length() == 0));
            if (read == null) {
            	br.close();
            	br = null;
            } else {
                currLineUA = false;
                int commentIndex = read.indexOf("#");
                if (commentIndex > -1) {
                    // Strip trailing comment
                    read = read.substring(0, commentIndex);
                }
                read = read.trim();
                Matcher uaMatcher = USER_AGENT_PATTERN.matcher(read);
                if (uaMatcher.matches()) {
                    String ua = uaMatcher.group(1).trim().toLowerCase();
                    if (current == null || current.size() != 0 || allowRuleFound || !lastLineUA) {
                        // only create new rules-list if necessary
                        // otherwise share with previous user-agent
                        current = new ArrayList<String>();
                    }
                    rules.put(ua, current);
                    allowRuleFound = false;
                    currLineUA = true;
                    LOGGER.fine("Found User-agent(" + ua + ") rules...");
                    continue;
                }
                Matcher disallowMatcher = DISALLOW_PATTERN.matcher(read);
                if (disallowMatcher.matches()) {
                	if (current == null) {
                        // buggy robots.txt
                    	bSyntaxErrors = true;
                        continue;
                    }
                    String path = disallowMatcher.group(1).trim();
					// Disallow: without path is just ignored.
					if (!path.isEmpty())
                    current.add(path);
                    continue;
                }
                Matcher allowMatcher = ALLOW_PATTERN.matcher(read);
                if (allowMatcher.matches()) {
                	// Mark that there was an allow rule to clear the current list for next user-agent
                	allowRuleFound = true;
                }
                // unknown line; do nothing for now
                
                // TODO: check for "Allow" lines, and flag a syntax error if 
                //       we encounter any unknown lines?
            }
        }
    }
1372207
private void processReader(Script s, BufferedReader reader, PrintWriter pw) throws IOException {
        String line;
        String lineCountName = "count";
        s.setProperty(lineCountName, BigInteger.ZERO);
        String autoSplitName = "split";
        s.setProperty("out", pw);

        try {
            InvokerHelper.invokeMethod(s, "begin", null);
        } catch (MissingMethodException mme) {
            // ignore the missing method exception
            // as it means no begin() method is present
        }

        while ((line = reader.readLine()) != null) {
            s.setProperty("line", line);
            s.setProperty(lineCountName, ((BigInteger)s.getProperty(lineCountName)).add(BigInteger.ONE));

            if(autoSplit) {
                s.setProperty(autoSplitName, line.split(splitPattern));
            }

            Object o = s.run();

            if (autoOutput && o != null) {
                pw.println(o);
            }
        }

        try {
            InvokerHelper.invokeMethod(s, "end", null);
        } catch (MissingMethodException mme) {
            // ignore the missing method exception
            // as it means no end() method is present
        }
    }
1353673
private void read(final File f) throws IOException {
        final FileReader fr = new FileReader(f);
        final BufferedReader br = new BufferedReader(fr);
        String input = null;
        while ((input = br.readLine()) != null) {

            // Trim whitespace.
            input = input.trim();

            // Skip comments
            if (input.startsWith(COMMENT)) continue;

            final int idx = input.indexOf(NAME_VALUE_DELIMITER);
            if (idx == -1) continue;

            String name, value;
            try {
                name = input.substring(0, idx);
                value = input.substring(idx + 1);
            } catch (IndexOutOfBoundsException e) {
                continue;
            }
            name = name.trim();
            value = value.trim();

            cfgMap.put(name, valueSubstitution(value));
        }
        br.close();
    }
1148482
private void fillReadBuffer() throws IOException {
		// special case for compressed files.
		if(splitLength == FileInputFormat.READ_WHOLE_SPLIT_FLAG) {
			int read = this.stream.read(this.readBuffer, 0, this.readBufferSize);
			if (read == -1) {
				exhausted = true;
			} else {
				this.streamPos += read;
				this.readBufferPos = 0;
				this.readBufferLimit = read;
			}
			return;
		}
		
		int toRead = (int) Math.min(this.streamEnd - this.streamPos, this.readBufferSize);
		if (toRead <= 0) {
			this.exhausted = true;
			return;
		}
		
		// fill read buffer
		int read = this.stream.read(this.readBuffer, 0, toRead);
		
		if (read <= 0) {
			this.exhausted = true;
		} else {
			this.streamPos += read;
			this.readBufferPos = 0;
			this.readBufferLimit = read;
		}
	}
820576
public static LogView parse(File file, Log log) throws IOException {
                LogView view = (LogView) xstream.fromXML(file);
                view.reinitialize();
                view.addTraces(log.getTraces());
                return view;
        }