Produce lemmas from a tokenized sentence and its postags.
@param tokens the tokens
@param posTags the pos tags
@return the lemmas
429962
public List<String> lemmatize(String[] tokens, String[] posTags) {
    String[] annotatedLemmas = lemmatizer.lemmatize(tokens, posTags);
    String[] decodedLemmas = lemmatizer.decodeLemmas(tokens, annotatedLemmas);
    final List<String> lemmas = new ArrayList<String>(Arrays.asList(decodedLemmas));
    return lemmas;
  }
197020
private Collection<AnnotationData> getAnnotationSet(final Collection<TagAndLength> pTags) {
		Collection<AnnotationData> ret = null;
		if (pTags != null) {
			Map<ITag, AnnotationData> data = AnnotationUtils.getInstance().getMap().get(getClass().getName());
			ret = new ArrayList<AnnotationData>(data.size());
			for (TagAndLength tal : pTags) {
				AnnotationData ann = data.get(tal.getTag());
				if (ann != null) {
					ann.setSize(tal.getLength() * BitUtils.BYTE_SIZE);
				} else {
					ann = new AnnotationData();
					ann.setSkip(true);
					ann.setSize(tal.getLength() * BitUtils.BYTE_SIZE);
				}
				ret.add(ann);
			}
		} else {
			ret = AnnotationUtils.getInstance().getMapSet().get(getClass().getName());
		}
		return ret;
	}
631578
int POSTAG(final List<Integer> postags, int id)
    {
        return ((id != -1) ? (postags.get(id) + kPostagInFeaturespace) : kNilPostag);
    }
283607
private List<String> getDictKeys(final String word, final String postag) {
    final List<String> keys = new ArrayList<String>();
    keys.addAll(Arrays.asList(word.toLowerCase(), postag));
    return keys;
  }
1287433
public final void postAggLabel(PostAggItem postAggItem) throws RecognitionException {
		Token id=null;

		try {
			// druidG.g:550:2: ( ( AS WS id= ID ) )
			// druidG.g:550:4: ( AS WS id= ID )
			{
			// druidG.g:550:4: ( AS WS id= ID )
			// druidG.g:550:5: AS WS id= ID
			{
			match(input,AS,FOLLOW_AS_in_postAggLabel3584); 
			match(input,WS,FOLLOW_WS_in_postAggLabel3586); 
			id=(Token)match(input,ID,FOLLOW_ID_in_postAggLabel3590); 
			}

			postAggItem.name = (id!=null?id.getText():null);
			}

		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
		}
	}
1094183
public PostBuilder withTags(Term... tags) {
        return withTags(Arrays.stream(tags).map(Term::getId).collect(toList()));
    }
1345402
@Override
  public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens, boolean ignoreCase) throws IOException {
  	List<AnalyzedTokenReadings> tokens = super.tag(sentenceTokens, ignoreCase);
    for (int i = 0; i < tokens.size(); i++) {
    	AnalyzedTokenReadings reading = tokens.get(i);
      if (reading != null &&
      		reading.getToken() != null &&
      		reading.getToken().contains("ss") &&
      		!reading.isTagged()) {
        AnalyzedTokenReadings replacementReading = lookup(reading.getToken().replace("ss", "ß"));
        if(replacementReading != null) {
          for(AnalyzedToken at : replacementReading.getReadings()) {
          	reading.addReading(new AnalyzedToken(reading.getToken(), at.getPOSTag(), at.getLemma()));
          }
        }
      }
    }
    return tokens;
  }
576908
private Map<String, ParseTreeNode> getTokens() throws GrammarException,
	    TreeException {
	Map<String, ParseTreeNode> tokens = new HashMap<String, ParseTreeNode>();
	ParseTreeNode tokensTree = parserTree.getChild("Tokens");
	ParseTreeNode tokenDefinitions = tokensTree.getChild("TokenDefinitions");
	for (ParseTreeNode tokenDefinition : tokenDefinitions
		.getChildren("TokenDefinition")) {
	    // identifier...
	    String identifier = tokenDefinition.getChild("IDENTIFIER")
		    .getText();
	    // token parts...
	    tokens.put(identifier, tokenDefinition);
	    // visibility...
	    ParseTreeNode visibilityAST = tokenDefinition.getChild("Visibility");
	    if (visibilityAST.hasChild("HIDE")) {
		tokenVisibility.put(identifier, Visibility.HIDDEN);
	    } else if (visibilityAST.hasChild("IGNORE")) {
		tokenVisibility.put(identifier, Visibility.IGNORED);
	    } else {
		tokenVisibility.put(identifier, Visibility.VISIBLE);
	    }
	}
	return tokens;
    }
376308
public boolean hasPosTagStartingWith(String posTag) {
    boolean found = false;
    for (AnalyzedToken reading : anTokReadings) {
      if (reading.getPOSTag() != null) {
        found = reading.getPOSTag().startsWith(posTag);
        if (found) {
          break;
        }
      }
    }
    return found;
  }
996497
public static String getNAFTagSet(final String postag, final String lang) {

    String tag = null;
    if (lang.equalsIgnoreCase("de")) {
      tag = mapGermanCoNLL09TagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("en")) {
      tag = mapEnglishPennTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("es")) {
      tag = mapSpanishAncoraTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("eu")) {
      tag = mapUDTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("gl")) {
      tag = mapGalicianCTAGTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("fr")) {
      tag = mapFrenchCCTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("it")) {
      tag = mapUDTagSetToNAF(postag);
    } else if (lang.equalsIgnoreCase("nl")) {
      tag = mapWotanTagSetToNAF(postag);
    } else {
      tag = "O";
    }
    return tag;
  }
840903
protected List<String> getTokens(String text){
		String[] tokens = text.toLowerCase().split("[\\s\\p{Punct}]");
		List<String> tokenList = new ArrayList<String>(tokens.length);
		for (int i = 0; i < tokens.length; i++){
			if (tokens[i].trim().length() < 1){
				continue;
			}
			tokenList.add(tokens[i].trim());
		}
		return tokenList;
	}
834045
public String[] lemmatize(final String[] tokens) {
    final Span[] origSpans = this.sequenceLabeler.tag(tokens);
    final Span[] seqSpans = SequenceLabelerME.dropOverlappingSpans(origSpans);
    // TODO work with Spans only
    final String[] decodedLemmas = StringUtils.decodeLemmas(tokens, seqSpans);
    return decodedLemmas;
  }
14355
public final List<Morpheme> getMorphemes(final String[] tokens, final String[] posTags) {
    final List<String> lemmas = lemmatize(tokens, posTags);
    final List<Morpheme> morphemes = getMorphemesFromStrings(tokens, posTags, lemmas);
    return morphemes;
  }
670610
@Override
  public String[] synthesize(AnalyzedToken token, String posTag)
      throws IOException {
    String aOrAn = aVsAnRule.suggestAorAn(token.getToken());
    if (ADD_DETERMINER.equals(posTag)) {
      return new String[] { aOrAn, "the " + token.getToken() };
    } else if (ADD_IND_DETERMINER.equals(posTag)) {
      return new String[] { aOrAn };
    }
    IStemmer synthesizer = createStemmer();
    List<WordData> wordData = synthesizer.lookup(token.getLemma() + "|" + posTag);
    List<String> wordForms = new ArrayList<>();
    for (WordData wd : wordData) {
      wordForms.add(wd.getStem().toString());
    }
    return wordForms.toArray(new String[wordForms.size()]);
  }
49402
public final String annotatePOSToCoNLL(final KAFDocument kaf)
      throws IOException {
    final StringBuilder sb = new StringBuilder();
    final List<List<WF>> sentences = kaf.getSentences();
    for (final List<WF> wfs : sentences) {

      final List<ixa.kaflib.Span<WF>> tokenSpans = new ArrayList<ixa.kaflib.Span<WF>>();
      List<Morpheme> morphemes = null;
      // Get an array of token forms from a list of WF objects.
      final String[] tokens = new String[wfs.size()];
      for (int i = 0; i < wfs.size(); i++) {
        tokens[i] = wfs.get(i).getForm();
        final List<WF> wfTarget = new ArrayList<WF>();
        wfTarget.add(wfs.get(i));
        tokenSpans.add(KAFDocument.newWFSpan(wfTarget));
      }
      if (this.multiwords) {
        final String[] multiWordTokens = this.multiWordMatcher
            .getTokensWithMultiWords(tokens);
        List<String> posTags = this.posTagger.posAnnotate(multiWordTokens);
        String[] posTagsArray = new String[posTags.size()];
        posTagsArray = posTags.toArray(posTagsArray);
        morphemes = this.lemmatizer.getMorphemes(multiWordTokens, posTagsArray);
        getMultiWordSpans(tokens, wfs, tokenSpans);
      } else {
        List<String> posTags = this.posTagger.posAnnotate(tokens);
        String[] posTagsArray = new String[posTags.size()];
        posTagsArray = posTags.toArray(posTagsArray);
        morphemes = this.lemmatizer.getMorphemes(tokens, posTagsArray);
      }
      for (int i = 0; i < morphemes.size(); i++) {
        final String posTag = morphemes.get(i).getTag();
        final String word = morphemes.get(i).getWord();
        if (this.dictag || this.multiwords) {
          final String dictPosTag = this.dictMorphoTagger.tag(word, posTag);
          morphemes.get(i).setTag(dictPosTag);
        }
        // dictionary lemmatizer overwrites probabilistic predictions
        // if lemma is not equal to word
        if (this.dictLemmatizer != null) {
          final String lemma = this.dictLemmatizer.apply(word, morphemes.get(i)
              .getTag());
          if (!lemma.equalsIgnoreCase("O")) {
            morphemes.get(i).setLemma(lemma);
          }
        }
        sb.append(word).append("\t").append(morphemes.get(i).getLemma())
            .append("\t").append(morphemes.get(i).getTag()).append("\n");
      }
      sb.append("\n");
    }
    return sb.toString();
  }
481404
public final void getAllTagsLemmasToNAF(final KAFDocument kaf) {
    final List<List<WF>> sentences = kaf.getSentences();
    for (final List<WF> wfs : sentences) {

      final List<ixa.kaflib.Span<WF>> tokenSpans = new ArrayList<ixa.kaflib.Span<WF>>();
      final String[] tokens = new String[wfs.size()];
      for (int i = 0; i < wfs.size(); i++) {
        tokens[i] = wfs.get(i).getForm();
        final List<WF> wfTarget = new ArrayList<WF>();
        wfTarget.add(wfs.get(i));
        tokenSpans.add(KAFDocument.newWFSpan(wfTarget));
      }
      
      String[][] allPosTags = this.posTagger.getAllPosTags(tokens);
      ListMultimap<String, String> morphMap = lemmatizer.getMultipleLemmas(tokens, allPosTags);
      
      for (int i = 0; i < tokens.length; i++) {
        final Term term = kaf.newTerm(tokenSpans.get(i));
        List<String> posLemmaValues = morphMap.get(tokens[i]);
        if (this.dictLemmatizer != null) {
          dictLemmatizer.getAllPosLemmas(tokens[i], posLemmaValues);
        }
        String allPosLemmasSet = StringUtils.getSetStringFromList(posLemmaValues);
        final String posId = Resources.getKafTagSet(allPosTags[0][i], lang);
        final String type = Resources.setTermType(posId);
        term.setType(type);
        term.setLemma(posLemmaValues.get(0).split("#")[1]);
        term.setPos(posId);
        term.setMorphofeat(allPosLemmasSet);
      }
    }
  }
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }