Creates the tokens.

@param statsTokenList
the stats token list
@param tokensData
the tokens data
@param docSet
the doc set
@throws IOException
Signals that an I/O exception has occurred.
337498
private static void createTokens(List<ComponentToken> statsTokenList,
      Map<Integer, Integer> tokensData, List<Integer> docSet)
      throws IOException {
    if (statsTokenList != null) {
      for (ComponentToken token : statsTokenList) {
        token.dataCollector.initNewList(1);
        Integer tmpValue;
        long[] values = new long[docSet.size()];
        int value;
        int number = 0;
        if (tokensData != null) {
          for (int docId : docSet) {
            tmpValue = tokensData.get(docId);
            value = tmpValue == null ? 0 : tmpValue.intValue();
            if (((token.minimumLong == null) || (value >= token.minimumLong))
                && ((token.maximumLong == null)
                    || (value <= token.maximumLong))) {
              values[number] = value;
              number++;
            }
          }
        }
        if (number > 0) {
          token.dataCollector.add(values, number);
        }
        token.dataCollector.closeNewList();
      }
    }
  }
1147907
@Override
    public Token createToken(Map<String, Object> tokenData) throws TokenCreationFailedException {
        TokenFactory tokenFactory = ltpaConfig.getTokenFactory();
        return tokenFactory.createToken(tokenData);
    }
606116
public Document fromTokens(@NonNull String... tokens) {
      return fromTokens(Arrays.asList(tokens), getDefaultLanguage());
   }
654181
private void tokenize(JCas jcas) {
		// read tokenized text to add tokens to the jcas
		Logger.printDetail(component, "TreeTagger (tokenization) with: " + ttprops.abbFileName);
		
		EnumSet<Flag> flags = Flag.getSet(ttprops.languageSwitch);
		TreeTaggerTokenizer ttt; ttprops.abbFileName = "english-abbreviations";
		if(ttprops.abbFileName != null) {
			ttt = new TreeTaggerTokenizer(ttprops.rootPath + ttprops.fileSeparator + "lib" + ttprops.fileSeparator + ttprops.abbFileName, flags);
		} else {
			ttt = new TreeTaggerTokenizer(null, flags);
		}
		
		String docText = jcas.getDocumentText().replaceAll("\n\n", "\nEMPTYLINE\n");
		List<String> tokenized = ttt.tokenize(docText);
		
		int tokenOffset = 0;
		// loop through all the lines in the treetagger output
		for(String s : tokenized) {
			// charset missmatch fallback: signal (invalid) s
			if ((!(s.equals("EMPTYLINE"))) && (jcas.getDocumentText().indexOf(s, tokenOffset) < 0)) {
				Logger.printError(component, "Tokenization was interrupted because the token \"" + s 
						+ "\" could not be found in the original text. The reason for this might be "
						+ "that the encoding of the document is not UTF-8. This token was skipped and "
						+ "if it was part of a temporal expression, will not be extracted.");
				continue;
			}
			// create tokens and add them to the jcas's indexes.
			Token newToken = new Token(jcas);
			if (s.equals("EMPTYLINE")){
				newToken.setBegin(tokenOffset);
				newToken.setEnd(tokenOffset);
				newToken.setPos("EMPTYLINE");
				if (annotate_partofspeech){
					newToken.addToIndexes();
				}
			}
			else{
				newToken.setBegin(jcas.getDocumentText().indexOf(s, tokenOffset));
				newToken.setEnd(newToken.getBegin() + s.length());
				newToken.addToIndexes();
				tokenOffset = newToken.getEnd();
			}
		}
	}
230588
public Document createDocument(String[] tokenstring, String label) {
		this.lexicon.incrementDocCount();
		SimpleDocument doc = new SimpleDocument(tokenstring, this.lexicon, true);
		doc.setLabel(this.lexicon.getLabelIndex(label));
		return doc;
	}
230876
@Override
  public void addTokens(final byte[] tokens) {

    try (final DataInputBuffer buf = new DataInputBuffer()) {

      buf.reset(tokens, tokens.length);
      final Credentials credentials = new Credentials();
      credentials.readTokenStorageStream(buf);

      final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
      ugi.addCredentials(credentials);
      LOG.log(Level.FINEST, "Added {0} tokens for user {1}", new Object[] {credentials.numberOfTokens(), ugi});

    } catch (final IOException ex) {
      LOG.log(Level.SEVERE, "Could not access tokens in user credentials.", ex);
      throw new RuntimeException(ex);
    }
  }
237344
@Override
  public MtasTokenCollection createTokenCollection(Reader reader)
      throws MtasParserException, MtasConfigException {
    AtomicInteger position = new AtomicInteger(0);
    MtasCRMAncestors unknownAncestors = new MtasCRMAncestors();

    Map<String, Set<Integer>> idPositions = new HashMap<>();
    Map<String, Integer[]> idOffsets = new HashMap<>();

    Map<String, Map<Integer, Set<String>>> updateList = createUpdateList();
    Map<String, List<MtasParserObject>> currentList = createCurrentList();

    tokenCollection = new MtasTokenCollection();
    MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
    try (MtasBufferedReader br = new MtasBufferedReader(reader)) {
      String line;
      int currentOffset;
      int previousOffset = br.getPosition();
      MtasParserObject currentObject;
      Pattern headerPattern = Pattern.compile("^@ @ @(.*)$");
      Pattern regularPattern = Pattern.compile(
          "^([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+)$");
      Matcher matcherHeader;
      Matcher matcherRegular = null;
      Set<MtasParserObject> newPreviousSentence = new HashSet<>();
      Set<MtasParserObject> previousSentence = new HashSet<>();
      Set<MtasParserObject> newPreviousClause = new HashSet<>();
      Set<MtasParserObject> previousClause = new HashSet<>(); 
      String[] matcherList = new String[8];
      while ((line = br.readLine()) != null) {
        currentOffset = br.getPosition();
        matcherHeader = headerPattern.matcher(line.trim());
        matcherRegular = regularPattern.matcher(line.trim());
        if (matcherRegular.matches()) {
          newPreviousSentence.clear();
          matcherList = createMatcherList(matcherRegular); 
          for (int i = 4; i < 8; i++) {
            List<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<>();
            Set<MtasParserObject> tmpList = processCRMSentence(
                mtasTokenIdFactory, String.valueOf(i),
                matcherList[i], currentOffset,
                functionOutputList, unknownAncestors, currentList, updateList,
                idPositions, idOffsets, previousSentence, previousClause);
            if (tmpList != null) {
              newPreviousSentence.addAll(tmpList);
            }
            for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
              tmpList = processCRMSentence(mtasTokenIdFactory,
                  functionOutput.name, functionOutput.value, currentOffset,
                  functionOutputList, unknownAncestors, currentList, updateList,
                  idPositions, idOffsets, previousSentence, previousClause);
              if (tmpList != null) {
                newPreviousSentence.addAll(tmpList);
              }
            }
          }
          if (!newPreviousSentence.isEmpty()) {
            previousSentence.clear();
            previousSentence.addAll(newPreviousSentence);
          }
          newPreviousClause.clear();
          for (int i = 4; i < 8; i++) {
            ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<>();
            Set<MtasParserObject> tmpList = processCRMClause(mtasTokenIdFactory,
                String.valueOf(i), matcherList[i], currentOffset,
                functionOutputList, unknownAncestors, currentList, updateList,
                idPositions, idOffsets, previousClause);
            if (tmpList != null) {
              newPreviousClause.addAll(tmpList);
            }
            for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
              tmpList = processCRMClause(mtasTokenIdFactory,
                  functionOutput.name, functionOutput.value, currentOffset,
                  functionOutputList, unknownAncestors, currentList, updateList,
                  idPositions, idOffsets, previousClause);
              if (tmpList != null) {
                newPreviousClause.addAll(tmpList);
              }
            }
          }
          if (!newPreviousClause.isEmpty()) {
            previousClause.clear();
            previousClause.addAll(newPreviousClause);
          }
        }

        if (matcherRegular.matches() && !matcherHeader.matches()) {
          matcherRegular = regularPattern.matcher(line.trim());
          if (matcherRegular.matches()) {
            // regular line - start word
            currentObject = new MtasParserObject(wordType);
            currentObject.setOffsetStart(previousOffset);
            currentObject.setRealOffsetStart(previousOffset);
            currentObject.setUnknownAncestorNumber(unknownAncestors.unknown);
            if (!prevalidateObject(currentObject, currentList)) {
              unknownAncestors.unknown++;
            } else {
              int p = position.getAndIncrement();
              currentObject.addPosition(p);
              currentObject.objectId = "word_" + p;
              currentList.get(MAPPING_TYPE_WORD).add(currentObject);
              unknownAncestors.unknown = 0;
              // check for crmPair
              for (int i = 0; i < 8; i++) {
                List<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<>();
                processCRMPair(mtasTokenIdFactory, p, String.valueOf(i),
                    matcherList[i], currentOffset,
                    functionOutputList, unknownAncestors, currentList,
                    updateList, idPositions, idOffsets);
                for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
                  processCRMPair(mtasTokenIdFactory, p, functionOutput.name,
                      functionOutput.value, currentOffset, functionOutputList,
                      unknownAncestors, currentList, updateList, idPositions,
                      idOffsets);
                }
              }
              // compute word annotations
              for (int i = 0; i < 8; i++) {
                ArrayList<MtasCRMParserFunctionOutput> functionOutputList = new ArrayList<>();
                functionOutputList
                    .addAll(processWordAnnotation(mtasTokenIdFactory,
                        String.valueOf(i), matcherList[i],
                        previousOffset, currentOffset, unknownAncestors,
                        currentList, updateList, idPositions, idOffsets));
                for (MtasCRMParserFunctionOutput functionOutput : functionOutputList) {
                  processWordAnnotation(mtasTokenIdFactory, functionOutput.name,
                      functionOutput.value, previousOffset, currentOffset,
                      unknownAncestors, currentList, updateList, idPositions,
                      idOffsets);
                }
              }
            }
            // finish word
            if (unknownAncestors.unknown > 0) {
              unknownAncestors.unknown--;
            } else {
              currentObject = currentList.get(MAPPING_TYPE_WORD)
                  .remove(currentList.get(MAPPING_TYPE_WORD).size() - 1);
              assert unknownAncestors.unknown == 0 : "error in administration "
                  + currentObject.getType().getName();
              currentObject.setText(null);
              currentObject.setOffsetEnd(currentOffset - 1);
              currentObject.setRealOffsetEnd(currentOffset - 1);
              // update ancestor groups with position and offset
              for (MtasParserObject currentGroup : currentList
                  .get(MAPPING_TYPE_GROUP)) {
                currentGroup.addPositions(currentObject.getPositions());
                currentGroup.addOffsetStart(currentObject.getOffsetStart());
                currentGroup.addOffsetEnd(currentObject.getOffsetEnd());
              }
              idPositions.put(currentObject.getId(),
                  currentObject.getPositions());
              idOffsets.put(currentObject.getId(), currentObject.getOffset());
              currentObject.updateMappings(idPositions, idOffsets);
              unknownAncestors.unknown = currentObject
                  .getUnknownAncestorNumber();
              computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                  currentList, updateList);
            }

          } else {
            // System.out.println("PROBLEM: " + line);
          }
        }
        previousOffset = br.getPosition();
      }
      closePrevious(mtasTokenIdFactory, previousSentence, previousOffset,
          unknownAncestors, currentList, updateList, idPositions, idOffsets);
      closePrevious(mtasTokenIdFactory, previousClause, previousOffset,
          unknownAncestors, currentList, updateList, idPositions, idOffsets);
    } catch (IOException e) {
      log.debug(e);
      throw new MtasParserException(e.getMessage());
    }
    // final check
    tokenCollection.check(autorepair, makeunique);
    return tokenCollection;

  }
242066
private function createToken($jsonData)
    {
        $content = json_decode($jsonData, true);

        if (!is_array($content)) {
            throw new TokenException(sprintf('Could not decode JSON string: "%s"', $jsonData));
        }

        return Token::create($content);
    }
359816
public void setSyntaxTokens(java.util.Collection<SyntaxToken> syntaxTokens) {
        if (syntaxTokens == null) {
            this.syntaxTokens = null;
            return;
        }

        this.syntaxTokens = new java.util.ArrayList<SyntaxToken>(syntaxTokens);
    }
968186
final String toTokenString() throws IOException {
    String[] stringToFormat = toFinalString(null);
    return String.join("|", Arrays.asList(stringToFormat));
  }
1010232
@Override
  public MtasTokenCollection createTokenCollection(Reader reader)
      throws MtasParserException, MtasConfigException {
    Boolean hasRoot = rootTag == null ? true : false;
    Boolean parsingContent = contentTag == null ? true : false;
    String textContent = null;
    Integer unknownAncestors = 0;
    Integer lastOffset = 0;

    AtomicInteger position = new AtomicInteger(0);
    Map<String, Set<Integer>> idPositions = new HashMap<>();
    Map<String, Integer[]> idOffsets = new HashMap<>();

    Map<String, Map<Integer, Set<String>>> updateList = createUpdateList();
    Map<String, List<MtasParserObject>> currentList = createCurrentList();
    Map<String, Map<String, String>> variables = createVariables();

    tokenCollection = new MtasTokenCollection();
    MtasTokenIdFactory mtasTokenIdFactory = new MtasTokenIdFactory();
    XMLInputFactory factory = XMLInputFactory.newInstance();
    try {      
      XMLStreamReader streamReader = factory.createXMLStreamReader(reader);
      QName qname;
      try {
        int event = streamReader.getEventType();
        MtasParserType<?> currentType;
        MtasParserType<?> tmpCurrentType;
        MtasParserType<?> tmpVariableType;
        MtasParserObject currentObject = null;
        MtasParserObject variableObject = null;
        while (true) {
          switch (event) {
          case XMLStreamConstants.START_DOCUMENT:
            log.debug("start of document");
            String encodingScheme = streamReader.getCharacterEncodingScheme();
            if (encodingScheme == null) {
              //ignore for now
              log.info("No encodingScheme found, assume utf-8");
              //throw new MtasParserException("No encodingScheme found");              
            } else if (!encodingScheme.equalsIgnoreCase("utf-8")) {
              throw new MtasParserException(
                  "XML not UTF-8 encoded but '" + encodingScheme + "'");
            }
            break;
          case XMLStreamConstants.END_DOCUMENT:
            log.debug("end of document");
            break;
          case XMLStreamConstants.SPACE:
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            break;
          case XMLStreamConstants.START_ELEMENT:
            // get data
            qname = streamReader.getName();
            // check for rootTag
            if (!hasRoot) {
              if (qname.equals(getQName(rootTag))) {
                hasRoot = true;
              } else {
                throw new MtasParserException("No " + rootTag);
              }
              // parse content
            } else {
              if ((tmpVariableType = variableTypes.get(qname)) != null) {
                variableObject = new MtasParserObject(tmpVariableType);
                collectAttributes(variableObject, streamReader);
                computeVariablesFromObject(variableObject, currentList,
                    variables);
              }
              if (parsingContent) {
                // check for relation : not within word, not within
                // groupAnnotation
                if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (tmpCurrentType = relationTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_RELATION).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for relation annotation: not within word, but within
                  // relation
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (!currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (tmpCurrentType = relationAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for group: not within word, not within relation, not
                  // within groupAnnotation
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (tmpCurrentType = groupTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_GROUP).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for group annotation: not within word, not within
                  // relation, but within group
                } else if ((currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (!currentList.get(MAPPING_TYPE_GROUP).isEmpty())
                    && (tmpCurrentType = groupAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for word: not within relation, not within
                  // groupAnnotation, not within word, not within wordAnnotation
                } else if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (currentList.get(MAPPING_TYPE_WORD_ANNOTATION).isEmpty())
                    && (tmpCurrentType = wordTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setOffsetStart(lastOffset);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentObject.addPosition(position.getAndIncrement());
                    currentList.get(MAPPING_TYPE_WORD).add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for word annotation: not within relation, not within
                  // groupAnnotation, but within word
                } else if ((currentList.get(MAPPING_TYPE_RELATION).isEmpty())
                    && (currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                        .isEmpty())
                    && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())
                    && (tmpCurrentType = wordAnnotationTypes
                        .get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.addPositions(currentList.get(MAPPING_TYPE_WORD)
                      .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                      .getPositions());
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                        .add(currentObject);
                    unknownAncestors = 0;
                  }
                  // check for references: within relation
                } else if (!currentList.get(MAPPING_TYPE_RELATION).isEmpty()
                    && (tmpCurrentType = refTypes.get(qname)) != null) {
                  currentObject = new MtasParserObject(tmpCurrentType);
                  collectAttributes(currentObject, streamReader);
                  currentObject.setUnknownAncestorNumber(unknownAncestors);
                  currentObject.setRealOffsetStart(lastOffset);
                  if (!prevalidateObject(currentObject, currentList)) {
                    unknownAncestors++;
                  } else {
                    currentType = tmpCurrentType;
                    currentList.get(MAPPING_TYPE_REF).add(currentObject);
                    unknownAncestors = 0;
                    // add reference to ancestor relations
                    for (MtasParserObject currentRelation : currentList
                        .get(MAPPING_TYPE_RELATION)) {
                      currentRelation.addRefId(currentObject
                          .getAttribute(currentType.getRefAttributeName()));
                      // register mapping for relation (for recursive relations)
                      SortedSet<String> keyMapList;
                      if (currentRelation.getId() != null) {
                        if (relationKeyMap
                            .containsKey(currentRelation.getId())) {
                          keyMapList = relationKeyMap
                              .get(currentRelation.getId());
                        } else {
                          keyMapList = new TreeSet<>();
                          relationKeyMap.put(currentRelation.getId(),
                              keyMapList);
                        }
                        keyMapList.add(currentObject
                            .getAttribute(currentType.getRefAttributeName()));
                      }
                    }
                  }
                } else {
                  unknownAncestors++;
                }
                // check for start content
              } else if (qname.equals(getQName(contentTag))) {
                parsingContent = true;
                // unexpected
              } else if (!allowNonContent) {
                throw new MtasParserException(
                    "Unexpected " + qname.getLocalPart() + " in document");
              }
            }
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            break;
          case XMLStreamConstants.END_ELEMENT:
            // set offset (end of end-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            // get data
            qname = streamReader.getName();
            // parse content
            if (parsingContent) {
              if (unknownAncestors > 0) {
                unknownAncestors--;
                // check for reference: because otherwise currentList should
                // contain no references
              } else if (!currentList.get(MAPPING_TYPE_REF).isEmpty()) {
                if ((currentType = refTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_REF)
                      .remove(currentList.get(MAPPING_TYPE_REF).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text and realOffset: not relevant
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for wordAnnotation: because otherwise currentList
                // should contain no wordAnnotations
              } else if (!currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = wordAnnotationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                      .remove(
                          currentList.get(MAPPING_TYPE_WORD_ANNOTATION).size()
                              - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  // offset always null, so update later with word (should be
                  // possible)
                  if ((currentObject.getId() != null)
                      && (!currentList.get(MAPPING_TYPE_WORD).isEmpty())) {
                    currentList.get(MAPPING_TYPE_WORD)
                        .get((currentList.get(MAPPING_TYPE_WORD).size() - 1))
                        .addUpdateableIdWithOffset(currentObject.getId());
                  }
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for word: because otherwise currentList should contain
                // no words
              } else if (!currentList.get(MAPPING_TYPE_WORD).isEmpty()) {
                if ((currentType = wordTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_WORD)
                      .remove(currentList.get(MAPPING_TYPE_WORD).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setOffsetEnd(lastOffset);
                  currentObject.setRealOffsetEnd(lastOffset);
                  // update ancestor groups with position and offset
                  for (MtasParserObject currentGroup : currentList
                      .get(MAPPING_TYPE_GROUP)) {
                    currentGroup.addPositions(currentObject.getPositions());
                    currentGroup.addOffsetStart(currentObject.getOffsetStart());
                    currentGroup.addOffsetEnd(currentObject.getOffsetEnd());
                  }
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for group annotation: because otherwise currentList
                // should contain no groupAnnotations
              } else if (!currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = groupAnnotationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                      .remove(
                          currentList.get(MAPPING_TYPE_GROUP_ANNOTATION).size()
                              - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for relation annotation
              } else if (!currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                  .isEmpty()) {
                if ((currentType = relationAnnotationTypes
                    .get(qname)) != null) {
                  currentObject = currentList
                      .get(MAPPING_TYPE_RELATION_ANNOTATION).remove(currentList
                          .get(MAPPING_TYPE_RELATION_ANNOTATION).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for relation
              } else if (!currentList.get(MAPPING_TYPE_RELATION).isEmpty()) {
                if ((currentType = relationTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_RELATION).remove(
                      currentList.get(MAPPING_TYPE_RELATION).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text: should not occur
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  // this shouldn't happen
                }
                // check for group
              } else if (!currentList.get(MAPPING_TYPE_GROUP).isEmpty()) {
                if ((currentType = groupTypes.get(qname)) != null) {
                  currentObject = currentList.get(MAPPING_TYPE_GROUP)
                      .remove(currentList.get(MAPPING_TYPE_GROUP).size() - 1);
                  assert currentObject.getType()
                      .equals(currentType) : "object expected to be "
                          + currentObject.getType().getName() + ", not "
                          + currentType.getName();
                  assert unknownAncestors == 0 : "error in administration "
                      + currentObject.getType().getName();
                  // ignore text: should not occur
                  currentObject.setRealOffsetEnd(lastOffset);
                  idPositions.put(currentObject.getId(),
                      currentObject.getPositions());
                  idOffsets.put(currentObject.getId(),
                      currentObject.getOffset());
                  currentObject.updateMappings(idPositions, idOffsets);
                  unknownAncestors = currentObject.getUnknownAncestorNumber();
                  computeMappingsFromObject(mtasTokenIdFactory, currentObject,
                      currentList, updateList);
                } else {
                  unknownAncestors--;
                }
              } else if (qname.equals(getQName("text"))) {
                parsingContent = false;
                assert unknownAncestors == 0 : "error in administration unknownAncestors";
                assert currentList.get(MAPPING_TYPE_REF)
                    .isEmpty() : "error in administration references";
                assert currentList.get(MAPPING_TYPE_GROUP)
                    .isEmpty() : "error in administration groups";
                assert currentList.get(MAPPING_TYPE_GROUP_ANNOTATION)
                    .isEmpty() : "error in administration groupAnnotations";
                assert currentList.get(MAPPING_TYPE_WORD)
                    .isEmpty() : "error in administration words";
                assert currentList.get(MAPPING_TYPE_WORD_ANNOTATION)
                    .isEmpty() : "error in administration wordAnnotations";
                assert currentList.get(MAPPING_TYPE_RELATION)
                    .isEmpty() : "error in administration relations";
                assert currentList.get(MAPPING_TYPE_RELATION_ANNOTATION)
                    .isEmpty() : "error in administration relationAnnotations";
              }
            }
            // forget text
            textContent = null;
            break;
          case XMLStreamConstants.CHARACTERS:
            // set offset (end of start-element)
            lastOffset = streamReader.getLocation().getCharacterOffset();
            // check for text
            if (streamReader.hasText()) {
              textContent = streamReader.getText();
            }
            if (currentObject != null && unknownAncestors.equals(0)) {
              currentObject.addText(textContent);
            }
            break;
          default:
            break;
          }
          if (!streamReader.hasNext()) {
            break;
          }
          event = streamReader.next();
        }
      } finally {
        streamReader.close();
      }
      // final checks
      assert unknownAncestors == 0 : "error in administration unknownAncestors";
      assert hasRoot : "no " + rootTag;
    } catch (XMLStreamException e) {
      log.debug(e);      
      throw new MtasParserException("No valid XML: " + e.getMessage());
    }

    // update tokens with variable
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_VARIABLE).entrySet()) {
      MtasToken token = tokenCollection.get(updateItem.getKey());
      String encodedPrefix = token.getPrefix();
      String encodedPostfix = token.getPostfix();
      token.setValue(decodeAndUpdateWithVariables(encodedPrefix, encodedPostfix,
          variables));
    }
    // update tokens with offset
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_OFFSET).entrySet()) {
      Set<String> refIdList = new HashSet<>();
      for (String refId : updateItem.getValue()) {
        if (idPositions.containsKey(refId)) {
          refIdList.add(refId);
        }
        if (relationKeyMap.containsKey(refId)) {
          refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
        }
      }
      for (String refId : refIdList) {
        Integer[] refOffset = idOffsets.get(refId);
        Integer tokenId = updateItem.getKey();
        if (tokenId != null && refOffset != null) {
          MtasToken token = tokenCollection.get(tokenId);
          token.addOffset(refOffset[0], refOffset[1]);
        }
      }
    }
    // update tokens with position
    for (Entry<Integer, Set<String>> updateItem : updateList
        .get(UPDATE_TYPE_POSITION).entrySet()) {
      HashSet<String> refIdList = new HashSet<>();
      for (String refId : updateItem.getValue()) {
        if (idPositions.containsKey(refId)) {
          refIdList.add(refId);
        }
        if (relationKeyMap.containsKey(refId)) {
          refIdList.addAll(recursiveCollect(refId, relationKeyMap, 10));
        }
      }
      for (String refId : refIdList) {
        Set<Integer> refPositions = idPositions.get(refId);
        Integer tokenId = updateItem.getKey();
        if (tokenId != null && refPositions != null) {
          MtasToken token = tokenCollection.get(tokenId);
          token.addPositions(refPositions);
        }
      }
    }
    // final check
    tokenCollection.check(autorepair, makeunique);
    return tokenCollection;
  }
523035
public void initFromAnnotation(final Data pData) {
		dateStandard = pData.dateStandard();
		format = pData.format();
		index = pData.index();
		readHexa = pData.readHexa();
		size = pData.size();
		if (pData.tag() != null) {
			tag = EmvTags.find(BytesUtils.fromString(pData.tag()));
		}
	}
184570
@Override
	public DataSet<Tuple2<VertexKey, VertexValue>> createResult() {
		if (this.initialVertices == null) {
			throw new IllegalStateException("The input data set has not been set.");
		}
		
		// prepare some type information
		TypeInformation<Tuple2<VertexKey, VertexValue>> vertexTypes = initialVertices.getType();
		TypeInformation<VertexKey> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
		TypeInformation<Tuple2<VertexKey, Message>> messageTypeInfo = new TupleTypeInfo<Tuple2<VertexKey,Message>>(keyType, messageType);		
		
		// set up the iteration operator
		final String name = (this.name != null) ? this.name :
			"Vertex-centric iteration (" + updateFunction + " | " + messagingFunction + ")";
		final int[] zeroKeyPos = new int[] {0};
	
		final DeltaIteration<Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>> iteration =
			this.initialVertices.iterateDelta(this.initialVertices, this.maximumNumberOfIterations, zeroKeyPos);
		iteration.name(name);
		iteration.parallelism(parallelism);
		
		// register all aggregators
		for (Map.Entry<String, Class<? extends Aggregator<?>>> entry : this.aggregators.entrySet()) {
			iteration.registerAggregator(entry.getKey(), entry.getValue());
		}
		
		// build the messaging function (co group)
		CoGroupOperator<?, ?, Tuple2<VertexKey, Message>> messages;
		if (edgesWithoutValue != null) {
			MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message> messenger = new MessagingUdfNoEdgeValues<VertexKey, VertexValue, Message>(messagingFunction, messageTypeInfo);
			messages = this.edgesWithoutValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
		}
		else {
			MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue> messenger = new MessagingUdfWithEdgeValues<VertexKey, VertexValue, Message, EdgeValue>(messagingFunction, messageTypeInfo);
			messages = this.edgesWithValue.coGroup(iteration.getWorkset()).where(0).equalTo(0).with(messenger);
		}
		
		// configure coGroup message function with name and broadcast variables
		messages = messages.name("Messaging");
		for (Tuple2<String, DataSet<?>> e : this.bcVarsMessaging) {
			messages = messages.withBroadcastSet(e.f1, e.f0);
		}
		
		VertexUpdateUdf<VertexKey, VertexValue, Message> updateUdf = new VertexUpdateUdf<VertexKey, VertexValue, Message>(updateFunction, vertexTypes);
		
		// build the update function (co group)
		CoGroupOperator<?, ?, Tuple2<VertexKey, VertexValue>> updates =
				messages.coGroup(iteration.getSolutionSet()).where(0).equalTo(0).with(updateUdf);
		
		// configure coGroup update function with name and broadcast variables
		updates = updates.name("Vertex State Updates");
		for (Tuple2<String, DataSet<?>> e : this.bcVarsUpdate) {
			updates = updates.withBroadcastSet(e.f1, e.f0);
		}

		// let the operator know that we preserve the key field
		updates.withConstantSetFirst("0").withConstantSetSecond("0");
		
		return iteration.closeWith(updates, updates);
		
	}
674227
public static void collectCollection(IndexReader reader, List<Integer> docSet,
      ComponentCollection collectionInfo) throws IOException {
    if (collectionInfo.action().equals(ComponentCollection.ACTION_CHECK)) {
      // can't do anything in lucene for check
    } else if (collectionInfo.action()
        .equals(ComponentCollection.ACTION_LIST)) {
      // can't do anything in lucene for list
    } else if (collectionInfo.action()
        .equals(ComponentCollection.ACTION_CREATE)) {
      BytesRef term = null;
      PostingsEnum postingsEnum = null;
      Integer docId;
      Integer termDocId = -1;
      Terms terms;
      LeafReaderContext lrc;
      LeafReader r;
      ListIterator<LeafReaderContext> iterator = reader.leaves().listIterator();
      while (iterator.hasNext()) {
        lrc = iterator.next();
        r = lrc.reader();
        for (String field : collectionInfo.fields()) {
          if ((terms = r.terms(field)) != null) {
            TermsEnum termsEnum = terms.iterator();
            while ((term = termsEnum.next()) != null) {
              Iterator<Integer> docIterator = docSet.iterator();
              postingsEnum = termsEnum.postings(postingsEnum,
                  PostingsEnum.NONE);
              termDocId = -1;
              while (docIterator.hasNext()) {
                docId = docIterator.next() - lrc.docBase;
                if ((docId >= termDocId) && ((docId.equals(termDocId))
                    || ((termDocId = postingsEnum.advance(docId))
                        .equals(docId)))) {
                  collectionInfo.addValue(term.utf8ToString());
                  break;
                }
                if (termDocId.equals(PostingsEnum.NO_MORE_DOCS)) {
                  break;
                }
              }
            }
          }
        }
      }
    }
  }
930203
private void addToken(final List<String> list, String tok) {
        if (StringUtils.isEmpty(tok)) {
            if (isIgnoreEmptyTokens()) {
                return;
            }
            if (isEmptyTokenAsNull()) {
                tok = null;
            }
        }
        list.add(tok);
    }
171468
public static Collection<Token<? extends TokenIdentifier>> readTokensFromFile(Path tokenFilePath,
      Configuration configuration) throws IOException {
    return Credentials.readTokenStorageFile(tokenFilePath, configuration).getAllTokens();
  }
769615
public static boolean fixTokenOffsets(CoreMap docAnnotation)
  {
    List<CoreLabel> docTokens = docAnnotation.get(CoreAnnotations.TokensAnnotation.class);
    List<CoreMap> sentences = docAnnotation.get(CoreAnnotations.SentencesAnnotation.class);
    int i = 0;
    CoreLabel curDocToken = docTokens.get(0);
    for (CoreMap sentence:sentences) {
      List<CoreLabel> sentTokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
      CoreLabel sentTokenFirst = sentTokens.get(0);
      while (curDocToken != sentTokenFirst) {
        i++;
        if (i >= docTokens.size()) { return false; }
        curDocToken = docTokens.get(i);
      }
      int sentTokenBegin = i;
      CoreLabel sentTokenLast = sentTokens.get(sentTokens.size()-1);
      while (curDocToken != sentTokenLast) {
        i++;
        if (i >= docTokens.size()) { return false; }
        curDocToken = docTokens.get(i);
      }
      int sentTokenEnd = i+1;
      sentence.set(CoreAnnotations.TokenBeginAnnotation.class, sentTokenBegin);
      sentence.set(CoreAnnotations.TokenEndAnnotation.class, sentTokenEnd);
    }
    return true;
  }