Provides tokenizer access for the analyzer.

@param fieldName     field to be tokenized
@param reader
426870
@Override
    protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
        return new TokenStreamComponents(new WhitespaceLowerCaseTokenizer(matchVersion, reader));
    }
381355
public final static String process(final Reader reader, final Decorator decorator) throws IOException
    {
        return process(reader, Configuration.builder().setDecorator(decorator).build());
    }
260684
private static void processText(NameFinder[] finders, String[] tags, BufferedReader input) throws IOException {
    String[][] finderTags = new String[finders.length][];
    Map[] prevTokenMaps = createPrevTokenMaps(finders); 
    for (String line = input.readLine(); null != line; line = input.readLine()) {
      if (line.equals("")) {
        clearPrevTokenMaps(prevTokenMaps);
        System.out.println();
        continue;
      }
      Span[] spans = tokenizeToSpans(line);
      String[] tokens = spansToStrings(spans,line);
      for (int fi = 0, fl = finders.length; fi < fl; fi++) {
        finderTags[fi] = finders[fi].find(tokens, prevTokenMaps[fi]);
        //System.err.println("EnglighNameFinder.processText: "+tags[fi] + " " + java.util.Arrays.asList(finderTags[fi]));
      }
      updatePrevTokenMaps(prevTokenMaps,tokens,finderTags);
      for (int ti = 0, tl = tokens.length; ti < tl; ti++) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          //check for end tags
          if (ti != 0) {
            if ((finderTags[fi][ti].equals(NameFinderME.START) || finderTags[fi][ti].equals(NameFinderME.OTHER)) && (finderTags[fi][ti - 1].equals(NameFinderME.START) || finderTags[fi][ti - 1].equals(NameFinderME.CONTINUE))) {
              System.out.print("</" + tags[fi] + ">");
            }
          }
        }
        if (ti > 0 && spans[ti - 1].getEnd() < spans[ti].getStart()) {
          System.out.print(line.substring(spans[ti - 1].getEnd(), spans[ti].getStart()));
        }
        //check for start tags
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][ti].equals(NameFinderME.START)) {
            System.out.print("<" + tags[fi] + ">");
          }
        }
        System.out.print(tokens[ti]);
      }
      //final end tags
      if (tokens.length != 0) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][tokens.length - 1].equals(NameFinderME.START) || finderTags[fi][tokens.length - 1].equals(NameFinderME.CONTINUE)) {
            System.out.print("</" + tags[fi] + ">");
          }
        }
      }
      if (tokens.length != 0) {
        if (spans[tokens.length - 1].getEnd() < line.length()) {
          System.out.print(line.substring(spans[tokens.length - 1].getEnd()));
        }
      }
      System.out.println();
    }
  }
116919
public<T> Param<List<T>> asAny (ParamReader<List<T>> reader, String name) {
        return getParam(reader, name);
    }
438058
public static UserAgentParser parse(final Reader input, final Collection<BrowsCapField> fields)
            throws IOException, ParseException {
        return new UserAgentFileParser(fields).parse(input);
    }
419444
public void readConfig(Reader reader) throws IOException {
		try {
			XMLStreamReader xr = XMLInputFactory.newInstance().createXMLStreamReader(reader);
			try {
				while (!xr.isStartElement()) {
					xr.next();
				}
				processStartElement(xr, "simon-configuration");
				while (true) {
					if (isStartTag(xr, "callback")) {
						manager.callback().addCallback(processCallback(xr));
					} else if (isStartTag(xr, "filter-callback")) {
						manager.callback().addCallback(processFilterCallback(xr));
					} else if (isStartTag(xr, "simon")) {
						processSimon(xr);
					} else {
						break;
					}
				}
				assertEndTag(xr, "simon-configuration");
			} finally {
				xr.close();
			}
		} catch (XMLStreamException e) {
			manager.callback().onManagerWarning(null, e);
		} catch (SimonException e) {
			manager.callback().onManagerWarning(e.getMessage(), e);
		}
	}
627511
public void addGroupBy(String fieldName)
    {
        if (fieldName != null)
        {
            _getGroupby().add(new FieldHelper(fieldName, false));
        }
    }
372439
private List<String> getFieldListFromHeader(BufferedReader reader) {
        // Read the first line and watch out for BOM at the front of the header.
        String header = null;
        try {
            header = reader.readLine();
        } catch (IOException e) {
            // Leave header null
        }
        if (Utils.isEmpty(header)) {
            logErrorThrow("No header found in file");
        }
        if (header.charAt(0) == '\uFEFF') {
            header = header.substring(1);
        }
        
        // Split around commas, though this will include spaces.
        String[] tokens = header.split(",");
        List<String> result = new ArrayList<String>();
        for (String token : tokens) {
            // If this field matches the _ID field, add it with that name.
            String fieldName = token.trim();
            if (fieldName.equals(m_config.id)) {
                result.add(CommonDefs.ID_FIELD);
            } else {
                result.add(token.trim());
            }
        }
        return result;
    }
1120116
public final static String process(final Reader reader, final Configuration configuration) throws IOException
    {
        final Processor p = new Processor(!(reader instanceof BufferedReader) ? new BufferedReader(reader) : reader,
                configuration);
        return p.process();
    }
1339848
public static void readField(JsonParser jsonParser,
                               String expectedFieldName) throws IOException {
    readToken(jsonParser, expectedFieldName, JsonToken.FIELD_NAME);
    String fieldName = jsonParser.getCurrentName();
    if (!fieldName.equals(expectedFieldName)) {
      foundUnknownField(fieldName, expectedFieldName);
    }
  }
973590
public void readDictionary(Reader reader, String name) {
        name = name.toLowerCase();
        logger.debug("Reading dictionary: ", name);
        if (!dictionaries.containsKey(name)) {
            try {
                Dictionary dictionary = Dictionary.unmarshal(reader);
                dictionaries.put(name, dictionary);
                logger.debug("  ... loaded and stored");
            } catch (Exception exception) {
                logger.error("Could not read dictionary: ", name);
                logger.debug(exception);
            }
        } else {
            logger.error("Dictionary already loaded: ", name);
        }
    }
1351781
public static List<Token> tokenize(String warpRawText, EvaluatorCompiler compiler) throws ExpressionCompileException {
    ArrayList<Token> tokens = new ArrayList<Token>();

    //simple state machine to iterate the text and break it up into chunks
    char[] characters = warpRawText.toCharArray();

    StringBuilder token = new StringBuilder();
    TokenizerState state = TokenizerState.READING_TEXT;
    for (int i = 0; i < characters.length; i++) {

      //test for start of an expression
      if (TokenizerState.READING_TEXT.equals(state)) {
        if ('$' == characters[i]) {
          if ('{' == characters[i + 1]) {
            //YES it is the start of an expr, so close up the existing token & start a new one
            if (token.length() > 0) {
              tokens.add(CompiledToken.text(token.toString()));
              token = new StringBuilder();
            }

            state = TokenizerState.READING_EXPRESSION;
          }
        }
      }

      //test for end of an expr
      if (TokenizerState.READING_EXPRESSION.equals(state)) {
        if ('}' == characters[i]) {
          //YES it is the end of the expr, so close it up and start a new token
          token.append(characters[i]);

          tokens.add(CompiledToken.expression(token.toString(), compiler));
          token = new StringBuilder();

          state = TokenizerState.READING_TEXT;
          continue;   //dont add the trailing } to the new text field
        }
      }

      //add characters to the token normally
      token.append(characters[i]);
    }

    //should never be in reading expr mode at this point
    if (TokenizerState.READING_EXPRESSION.equals(state))
      throw new IllegalStateException("Error. Expression was not terminated properly: " + token.toString());

    //add last token read if it has any content (is always text)
    if (token.length() > 0)
      tokens.add(CompiledToken.text(token.toString()));

    // Pack list capacity to size (saves memory).
    tokens.trimToSize();

    return tokens;
  }
316539
public void extract(@NotNull final RandomAccessReader reader, @NotNull final Metadata metadata)
    {
        extract(reader, metadata, 0);
    }
286259
def analyze(self, text, index=None, analyzer=None, tokenizer=None, filters=None, field=None):
        
        if filters is None:
            filters = []
        argsets = 0
        args = {}

        if analyzer:
            args['analyzer'] = analyzer
            argsets += 1
        if tokenizer or filters:
            if tokenizer:
                args['tokenizer'] = tokenizer
            if filters:
                args['filters'] = ','.join(filters)
            argsets += 1
        if field:
            args['field'] = field
            argsets += 1

        if argsets > 1:
            raise ValueError('Argument conflict: Specify either analyzer, tokenizer/filters or field')

        if field and index is None:
            raise ValueError('field can only be specified with an index')

        path = make_path(index, '_analyze')
        return self.conn._send_request('POST', path, text, args)
757440
private static Map<String, String> readPrefixDeclaration(BufferedReader reader) throws IOException {
        Map<String, String> prefixes = new HashMap<>();
        String line;
        while (!(line = reader.readLine()).isEmpty()) {
            String[] tokens = line.split("[\t| ]+");
            prefixes.put(tokens[0], tokens[1]);
        }
        return prefixes;
    }
197048
def tokenize(self, s):
        
        javabridge.call(self.jobject, "tokenize", "(Ljava/lang/String;)V", s)
        return TokenIterator(self)
584822
public static void loadFrom(Reader reader, Proto target) throws Exception
    {
        loadFrom(new ANTLRReaderStream(reader), target);
    }