Decodes the lemma from the word and the induced lemma class.

@param tokens
the array of tokens
@param preds
the predicted lemma classes
@return the array of decoded lemmas
188388
public static String[] decodeLemmas(final String[] tokens,
      final Span[] preds) {
    final List<String> lemmas = new ArrayList<>();
    for (final Span span : preds) {
      String lemma = decodeShortestEditScript(
          span.getCoveredText(tokens).toLowerCase(), span.getType());
      // System.err.println("-> DEBUG: " + toks[i].toLowerCase() + " " +
      // preds[i] + " " + lemma);
      if (lemma.length() == 0) {
        lemma = "_";
      }
      lemmas.add(lemma);
    }
    return lemmas.toArray(new String[lemmas.size()]);
  }
1127033
public void mTokens() throws RecognitionException {
        // BELScript.g:1:8: ( T__24 | T__25 | T__26 | T__27 | T__28 | T__29 | T__30 | T__31 | T__32 | T__33 | T__34 | T__35 | T__36 | T__37 | T__38 | T__39 | T__40 | T__41 | T__42 | T__43 | T__44 | T__45 | T__46 | T__47 | T__48 | T__49 | T__50 | T__51 | T__52 | T__53 | T__54 | T__55 | T__56 | T__57 | T__58 | T__59 | T__60 | T__61 | T__62 | T__63 | T__64 | T__65 | T__66 | T__67 | T__68 | T__69 | T__70 | T__71 | T__72 | T__73 | T__74 | T__75 | T__76 | T__77 | T__78 | T__79 | T__80 | T__81 | T__82 | T__83 | T__84 | T__85 | T__86 | T__87 | T__88 | T__89 | T__90 | T__91 | T__92 | T__93 | T__94 | T__95 | T__96 | T__97 | T__98 | T__99 | T__100 | T__101 | T__102 | T__103 | T__104 | T__105 | T__106 | T__107 | T__108 | T__109 | T__110 | T__111 | T__112 | T__113 | T__114 | T__115 | T__116 | T__117 | T__118 | T__119 | T__120 | T__121 | T__122 | T__123 | T__124 | T__125 | T__126 | T__127 | T__128 | T__129 | T__130 | DOCUMENT_COMMENT | STATEMENT_COMMENT | DOCUMENT_KEYWORD | STATEMENT_GROUP_KEYWORD | IDENT_LIST | VALUE_LIST | OBJECT_IDENT | QUOTED_VALUE | OPEN_PAREN | CLOSE_PAREN | NS_PREFIX | NEWLINE | WS )
        int alt17=120;
        alt17 = dfa17.predict(input);
        switch (alt17) {
            case 1 :
                // BELScript.g:1:10: T__24
                {
                mT__24(); 

                }
                break;
            case 2 :
                // BELScript.g:1:16: T__25
                {
                mT__25(); 

                }
                break;
            case 3 :
                // BELScript.g:1:22: T__26
                {
                mT__26(); 

                }
                break;
            case 4 :
                // BELScript.g:1:28: T__27
                {
                mT__27(); 

                }
                break;
            case 5 :
                // BELScript.g:1:34: T__28
                {
                mT__28(); 

                }
                break;
            case 6 :
                // BELScript.g:1:40: T__29
                {
                mT__29(); 

                }
                break;
            case 7 :
                // BELScript.g:1:46: T__30
                {
                mT__30(); 

                }
                break;
            case 8 :
                // BELScript.g:1:52: T__31
                {
                mT__31(); 

                }
                break;
            case 9 :
                // BELScript.g:1:58: T__32
                {
                mT__32(); 

                }
                break;
            case 10 :
                // BELScript.g:1:64: T__33
                {
                mT__33(); 

                }
                break;
            case 11 :
                // BELScript.g:1:70: T__34
                {
                mT__34(); 

                }
                break;
            case 12 :
                // BELScript.g:1:76: T__35
                {
                mT__35(); 

                }
                break;
            case 13 :
                // BELScript.g:1:82: T__36
                {
                mT__36(); 

                }
                break;
            case 14 :
                // BELScript.g:1:88: T__37
                {
                mT__37(); 

                }
                break;
            case 15 :
                // BELScript.g:1:94: T__38
                {
                mT__38(); 

                }
                break;
            case 16 :
                // BELScript.g:1:100: T__39
                {
                mT__39(); 

                }
                break;
            case 17 :
                // BELScript.g:1:106: T__40
                {
                mT__40(); 

                }
                break;
            case 18 :
                // BELScript.g:1:112: T__41
                {
                mT__41(); 

                }
                break;
            case 19 :
                // BELScript.g:1:118: T__42
                {
                mT__42(); 

                }
                break;
            case 20 :
                // BELScript.g:1:124: T__43
                {
                mT__43(); 

                }
                break;
            case 21 :
                // BELScript.g:1:130: T__44
                {
                mT__44(); 

                }
                break;
            case 22 :
                // BELScript.g:1:136: T__45
                {
                mT__45(); 

                }
                break;
            case 23 :
                // BELScript.g:1:142: T__46
                {
                mT__46(); 

                }
                break;
            case 24 :
                // BELScript.g:1:148: T__47
                {
                mT__47(); 

                }
                break;
            case 25 :
                // BELScript.g:1:154: T__48
                {
                mT__48(); 

                }
                break;
            case 26 :
                // BELScript.g:1:160: T__49
                {
                mT__49(); 

                }
                break;
            case 27 :
                // BELScript.g:1:166: T__50
                {
                mT__50(); 

                }
                break;
            case 28 :
                // BELScript.g:1:172: T__51
                {
                mT__51(); 

                }
                break;
            case 29 :
                // BELScript.g:1:178: T__52
                {
                mT__52(); 

                }
                break;
            case 30 :
                // BELScript.g:1:184: T__53
                {
                mT__53(); 

                }
                break;
            case 31 :
                // BELScript.g:1:190: T__54
                {
                mT__54(); 

                }
                break;
            case 32 :
                // BELScript.g:1:196: T__55
                {
                mT__55(); 

                }
                break;
            case 33 :
                // BELScript.g:1:202: T__56
                {
                mT__56(); 

                }
                break;
            case 34 :
                // BELScript.g:1:208: T__57
                {
                mT__57(); 

                }
                break;
            case 35 :
                // BELScript.g:1:214: T__58
                {
                mT__58(); 

                }
                break;
            case 36 :
                // BELScript.g:1:220: T__59
                {
                mT__59(); 

                }
                break;
            case 37 :
                // BELScript.g:1:226: T__60
                {
                mT__60(); 

                }
                break;
            case 38 :
                // BELScript.g:1:232: T__61
                {
                mT__61(); 

                }
                break;
            case 39 :
                // BELScript.g:1:238: T__62
                {
                mT__62(); 

                }
                break;
            case 40 :
                // BELScript.g:1:244: T__63
                {
                mT__63(); 

                }
                break;
            case 41 :
                // BELScript.g:1:250: T__64
                {
                mT__64(); 

                }
                break;
            case 42 :
                // BELScript.g:1:256: T__65
                {
                mT__65(); 

                }
                break;
            case 43 :
                // BELScript.g:1:262: T__66
                {
                mT__66(); 

                }
                break;
            case 44 :
                // BELScript.g:1:268: T__67
                {
                mT__67(); 

                }
                break;
            case 45 :
                // BELScript.g:1:274: T__68
                {
                mT__68(); 

                }
                break;
            case 46 :
                // BELScript.g:1:280: T__69
                {
                mT__69(); 

                }
                break;
            case 47 :
                // BELScript.g:1:286: T__70
                {
                mT__70(); 

                }
                break;
            case 48 :
                // BELScript.g:1:292: T__71
                {
                mT__71(); 

                }
                break;
            case 49 :
                // BELScript.g:1:298: T__72
                {
                mT__72(); 

                }
                break;
            case 50 :
                // BELScript.g:1:304: T__73
                {
                mT__73(); 

                }
                break;
            case 51 :
                // BELScript.g:1:310: T__74
                {
                mT__74(); 

                }
                break;
            case 52 :
                // BELScript.g:1:316: T__75
                {
                mT__75(); 

                }
                break;
            case 53 :
                // BELScript.g:1:322: T__76
                {
                mT__76(); 

                }
                break;
            case 54 :
                // BELScript.g:1:328: T__77
                {
                mT__77(); 

                }
                break;
            case 55 :
                // BELScript.g:1:334: T__78
                {
                mT__78(); 

                }
                break;
            case 56 :
                // BELScript.g:1:340: T__79
                {
                mT__79(); 

                }
                break;
            case 57 :
                // BELScript.g:1:346: T__80
                {
                mT__80(); 

                }
                break;
            case 58 :
                // BELScript.g:1:352: T__81
                {
                mT__81(); 

                }
                break;
            case 59 :
                // BELScript.g:1:358: T__82
                {
                mT__82(); 

                }
                break;
            case 60 :
                // BELScript.g:1:364: T__83
                {
                mT__83(); 

                }
                break;
            case 61 :
                // BELScript.g:1:370: T__84
                {
                mT__84(); 

                }
                break;
            case 62 :
                // BELScript.g:1:376: T__85
                {
                mT__85(); 

                }
                break;
            case 63 :
                // BELScript.g:1:382: T__86
                {
                mT__86(); 

                }
                break;
            case 64 :
                // BELScript.g:1:388: T__87
                {
                mT__87(); 

                }
                break;
            case 65 :
                // BELScript.g:1:394: T__88
                {
                mT__88(); 

                }
                break;
            case 66 :
                // BELScript.g:1:400: T__89
                {
                mT__89(); 

                }
                break;
            case 67 :
                // BELScript.g:1:406: T__90
                {
                mT__90(); 

                }
                break;
            case 68 :
                // BELScript.g:1:412: T__91
                {
                mT__91(); 

                }
                break;
            case 69 :
                // BELScript.g:1:418: T__92
                {
                mT__92(); 

                }
                break;
            case 70 :
                // BELScript.g:1:424: T__93
                {
                mT__93(); 

                }
                break;
            case 71 :
                // BELScript.g:1:430: T__94
                {
                mT__94(); 

                }
                break;
            case 72 :
                // BELScript.g:1:436: T__95
                {
                mT__95(); 

                }
                break;
            case 73 :
                // BELScript.g:1:442: T__96
                {
                mT__96(); 

                }
                break;
            case 74 :
                // BELScript.g:1:448: T__97
                {
                mT__97(); 

                }
                break;
            case 75 :
                // BELScript.g:1:454: T__98
                {
                mT__98(); 

                }
                break;
            case 76 :
                // BELScript.g:1:460: T__99
                {
                mT__99(); 

                }
                break;
            case 77 :
                // BELScript.g:1:466: T__100
                {
                mT__100(); 

                }
                break;
            case 78 :
                // BELScript.g:1:473: T__101
                {
                mT__101(); 

                }
                break;
            case 79 :
                // BELScript.g:1:480: T__102
                {
                mT__102(); 

                }
                break;
            case 80 :
                // BELScript.g:1:487: T__103
                {
                mT__103(); 

                }
                break;
            case 81 :
                // BELScript.g:1:494: T__104
                {
                mT__104(); 

                }
                break;
            case 82 :
                // BELScript.g:1:501: T__105
                {
                mT__105(); 

                }
                break;
            case 83 :
                // BELScript.g:1:508: T__106
                {
                mT__106(); 

                }
                break;
            case 84 :
                // BELScript.g:1:515: T__107
                {
                mT__107(); 

                }
                break;
            case 85 :
                // BELScript.g:1:522: T__108
                {
                mT__108(); 

                }
                break;
            case 86 :
                // BELScript.g:1:529: T__109
                {
                mT__109(); 

                }
                break;
            case 87 :
                // BELScript.g:1:536: T__110
                {
                mT__110(); 

                }
                break;
            case 88 :
                // BELScript.g:1:543: T__111
                {
                mT__111(); 

                }
                break;
            case 89 :
                // BELScript.g:1:550: T__112
                {
                mT__112(); 

                }
                break;
            case 90 :
                // BELScript.g:1:557: T__113
                {
                mT__113(); 

                }
                break;
            case 91 :
                // BELScript.g:1:564: T__114
                {
                mT__114(); 

                }
                break;
            case 92 :
                // BELScript.g:1:571: T__115
                {
                mT__115(); 

                }
                break;
            case 93 :
                // BELScript.g:1:578: T__116
                {
                mT__116(); 

                }
                break;
            case 94 :
                // BELScript.g:1:585: T__117
                {
                mT__117(); 

                }
                break;
            case 95 :
                // BELScript.g:1:592: T__118
                {
                mT__118(); 

                }
                break;
            case 96 :
                // BELScript.g:1:599: T__119
                {
                mT__119(); 

                }
                break;
            case 97 :
                // BELScript.g:1:606: T__120
                {
                mT__120(); 

                }
                break;
            case 98 :
                // BELScript.g:1:613: T__121
                {
                mT__121(); 

                }
                break;
            case 99 :
                // BELScript.g:1:620: T__122
                {
                mT__122(); 

                }
                break;
            case 100 :
                // BELScript.g:1:627: T__123
                {
                mT__123(); 

                }
                break;
            case 101 :
                // BELScript.g:1:634: T__124
                {
                mT__124(); 

                }
                break;
            case 102 :
                // BELScript.g:1:641: T__125
                {
                mT__125(); 

                }
                break;
            case 103 :
                // BELScript.g:1:648: T__126
                {
                mT__126(); 

                }
                break;
            case 104 :
                // BELScript.g:1:655: T__127
                {
                mT__127(); 

                }
                break;
            case 105 :
                // BELScript.g:1:662: T__128
                {
                mT__128(); 

                }
                break;
            case 106 :
                // BELScript.g:1:669: T__129
                {
                mT__129(); 

                }
                break;
            case 107 :
                // BELScript.g:1:676: T__130
                {
                mT__130(); 

                }
                break;
            case 108 :
                // BELScript.g:1:683: DOCUMENT_COMMENT
                {
                mDOCUMENT_COMMENT(); 

                }
                break;
            case 109 :
                // BELScript.g:1:700: STATEMENT_COMMENT
                {
                mSTATEMENT_COMMENT(); 

                }
                break;
            case 110 :
                // BELScript.g:1:718: DOCUMENT_KEYWORD
                {
                mDOCUMENT_KEYWORD(); 

                }
                break;
            case 111 :
                // BELScript.g:1:735: STATEMENT_GROUP_KEYWORD
                {
                mSTATEMENT_GROUP_KEYWORD(); 

                }
                break;
            case 112 :
                // BELScript.g:1:759: IDENT_LIST
                {
                mIDENT_LIST(); 

                }
                break;
            case 113 :
                // BELScript.g:1:770: VALUE_LIST
                {
                mVALUE_LIST(); 

                }
                break;
            case 114 :
                // BELScript.g:1:781: OBJECT_IDENT
                {
                mOBJECT_IDENT(); 

                }
                break;
            case 115 :
                // BELScript.g:1:794: QUOTED_VALUE
                {
                mQUOTED_VALUE(); 

                }
                break;
            case 116 :
                // BELScript.g:1:807: OPEN_PAREN
                {
                mOPEN_PAREN(); 

                }
                break;
            case 117 :
                // BELScript.g:1:818: CLOSE_PAREN
                {
                mCLOSE_PAREN(); 

                }
                break;
            case 118 :
                // BELScript.g:1:830: NS_PREFIX
                {
                mNS_PREFIX(); 

                }
                break;
            case 119 :
                // BELScript.g:1:840: NEWLINE
                {
                mNEWLINE(); 

                }
                break;
            case 120 :
                // BELScript.g:1:848: WS
                {
                mWS(); 

                }
                break;

        }

    }
29493
public List<List<String>> decodeLabeledSequences(List<List<SeqElement>> seqs) {
		List<List<List<String>>> n1 = new ArrayList<List<List<String>>>();
		for (List<SeqElement> seq : seqs) {
			List<List<String>> n2 = new ArrayList<List<String>>();
			for (SeqElement el : seq) {
				n2.add(el.getFeatures());
			}
			n1.add(n2);
		}
		return decoder.decodeFromJava(n1);
	}
606116
public Document fromTokens(@NonNull String... tokens) {
      return fromTokens(Arrays.asList(tokens), getDefaultLanguage());
   }
14355
public final List<Morpheme> getMorphemes(final String[] tokens, final String[] posTags) {
    final List<String> lemmas = lemmatize(tokens, posTags);
    final List<Morpheme> morphemes = getMorphemesFromStrings(tokens, posTags, lemmas);
    return morphemes;
  }
724379
protected function parseTypeModifiers(array $tokens)
	{
		$modifiers = 0;
	
		foreach($tokens as $tok)
		{
			if(is_array($tok))
			{
				switch($tok[0])
				{
					case T_ABSTRACT:
						$modifiers |= MetaInfo::IS_ABSTRACT;
						break;
					case T_FINAL:
						$modifiers |= MetaInfo::IS_FINAL;
						break;
				}
			}
		}
	
		return $modifiers;
	}
619301
public List<String> getFirstSenseBio(final List<String> lemmas,
      final Span[] posTags) {

    final List<String> mostFrequentSenseList = new ArrayList<String>();

    String prefix = "-" + BioCodec.START;
    String mostFrequentSense = null;
    String searchSpan = null;
    // iterative over lemmas from the beginning
    for (int i = 0; i < lemmas.size(); i++) {
      mostFrequentSense = null;
      final String pos = posTags[i].getType();
      int j;
      // iterate over lemmas from the end
      for (j = lemmas.size() - 1; j >= i; j--) {
        // create span for search in multimap; the first search takes as span
        // the whole sentence
        final String endPos = posTags[j].getType();
        searchSpan = createSpan(lemmas, i, j);
        final String firstSpan = (searchSpan + "#" + pos.substring(0, 1))
            .toLowerCase();
        final TreeMultimap<Integer, String> mfsMap = getOrderedMap(firstSpan);
        if (!mfsMap.isEmpty()) {
          mostFrequentSense = getMFS(mfsMap);
          break;
        }
        final String lastSpan = (searchSpan + "#" + endPos.substring(0, 1))
            .toLowerCase();
        final TreeMultimap<Integer, String> mfsMapEnd = getOrderedMap(lastSpan);
        if (!mfsMapEnd.isEmpty()) {
          mostFrequentSense = getMFS(mfsMapEnd);
          break;
        }
      }
      prefix = "-" + BioCodec.START;
      // multi-token case
      if (mostFrequentSense != null) {
        while (i < j) {
          mostFrequentSenseList.add((mostFrequentSense + prefix).intern());
          prefix = "-" + BioCodec.CONTINUE;
          i++;
        }
      }
      // one word case or last member of multispan
      if (mostFrequentSense != null) {
        mostFrequentSenseList.add((mostFrequentSense + prefix).intern());
      } else {
        mostFrequentSenseList.add(BioCodec.OTHER);
      }
    }
    return mostFrequentSenseList;
  }
1204305
@Override
    protected List<Token> initializeTokens( List<Token> tokens ) {
        // THIS IS WHERE WE DO THE WORK OF PRE-PARSING TOKENS AND REPLACING KEYWORDS AND STATEMENT STARTS WITH
        // APPLICABLE TOKEN TYPE BITMASK VALUES
        // MyClass[] array = (MyClass[])list.toArray(new MyClass[list.size()]);

        Token[] tokensArray = tokens.toArray(new Token[tokens.size()]);
        List<Token> reTypedTokens = new ArrayList<Token>(tokens.size());

        for (int i = 0; i < tokensArray.length; i++) {
            boolean isStatementStart = false;
            if (isKeyWord(tokensArray[i].value())) {
                Token retypedToken = tokensArray[i].withType(DdlTokenizer.KEYWORD);
                // Now we check to see if this keyword begins a registered statement start

                // Keep track of token increment (# of tokens for a phrase)
                // Need to increment iterator (i) in case phrases like "ALTER ROLLBACK" appear. ROLLBACK is also a statement
                // start phrase and we need to walk ignore ROLLBACK in this case.
                int tokenIncrement = 0;
                for (String[] nextStmtStart : registeredStatementStartPhrases) {
                    boolean matches = true;

                    for (int j = 0; j < nextStmtStart.length; j++) {
                        if (matches) {
                            matches = nextStmtStart[j].equalsIgnoreCase(tokensArray[i + j].value())
                                      || nextStmtStart[j].equals(ANY_VALUE);
                        }
                    }
                    if (matches) {
                        isStatementStart = true;
                        tokenIncrement = nextStmtStart.length - 1;
                        break;
                    }
                }
                if (isStatementStart) {
                    retypedToken = retypedToken.withType(DdlTokenizer.STATEMENT_KEY);
                }
                reTypedTokens.add(retypedToken);

                if (isStatementStart) {
                    // Copy any additional tokens used in the phrase
                    for (int k = 0; k < tokenIncrement; k++) {
                        i++;
                        reTypedTokens.add(tokensArray[i]);
                    }
                }
            } else {
                reTypedTokens.add(tokensArray[i]);
            }

        }

        return reTypedTokens;
    }
835396
public function parse(array $tokens)
    {
        $symbolNodes = $this->detectSymbols($tokens);

        $nodes = $this->createTreeByBrackets($symbolNodes);

        $nodes = $this->transformTreeByFunctions($nodes);

        $this->checkGrammar($nodes);

        // Wrap the nodes in an array node.
        $rootNode = new ContainerNode($nodes);

        return $rootNode;
    }
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
891114
def lookup(self, tokens):
        """"""
        lemmatized_tokens = []
        if type(tokens) == list:
            for token in tokens:
                # look for token in lemma dict keys
                if token.lower() in self.lemmata.keys():
                    # `lemmas` is a list of possible lemmata. Probability values must be assigned.
                    # `lemmalist` is a list of the form [(LEMMA, PROBABILITY), (LEMMA, PROBABILITY)]
                    # `lemmaobj` is a tuple with the form (LEMMA, LIST)
                    lemmas = self.lemmata[token.lower()]
                    lemmalist = []
                    for lemma in lemmas:
                        lemmalist.append((lemma, 1/len(lemmas)))
                    lemmaobj = (token, lemmalist)
                else:
                # if token not found in lemma-headword list, return the token itself
                    lemmalist = []
                    lemmalist.append((token, 1))
                    lemmaobj = (token, lemmalist)
                lemmatized_tokens.append(lemmaobj)
        if type(tokens) == str:
            if tokens.lower() in self.lemmata.keys():
                # `lemmas` is a list of possible lemmata. Probability values must be assigned.
                # `lemmalist` is a list of the form [(LEMMA, PROBABILITY), (LEMMA, PROBABILITY)]
                # `lemmaobj` is a tuple with the form (LEMMA, LIST)
                lemmas = self.lemmata[tokens.lower()]
                lemmalist = []
                for lemma in lemmas:
                    lemmalist.append((lemma, 1/len(lemmas)))
                lemmaobj = (tokens, lemmalist)
            else:
            # if token not found in lemma-headword list, return the token itself
                lemmalist = []
                lemmalist.append((tokens, 1))
                lemmaobj = (tokens, lemmalist)
            lemmatized_tokens.append(lemmaobj)
        return lemmatized_tokens
1352541
public static String getStringFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String tok : tokens) {
      sb.append(tok).append(" ");
    }
    return sb.toString().trim();
  }
233203
protected function detectSymbols(array $tokens)
    {
        $symbolNodes = [];

        $expectingOpeningBracket = false; // True if we expect an opening bracket (after a function name)
        $openBracketCounter = 0;

        foreach ($tokens as $token) {
            $type = $token->getType();

            if ($type == Token::TYPE_WORD) {
                $identifier = $token->getValue();
                $symbol = $this->symbolContainer->find($identifier);

                if ($symbol === null) {
                    $this->throwException(
                        NotFoundException::class,
                        'Error: Detected unknown or invalid string identifier.',
                        $token->getPosition(),
                        $identifier
                    );
                }
            } elseif ($type == Token::TYPE_NUMBER) {
                // Notice: Numbers do not have an identifier
                $symbol = $this->symbolContainer->findSubtypes(Number::class)[0];
            } else { // Type Token::TYPE_CHARACTER:
                $identifier = $token->getValue();
                $symbol = $this->symbolContainer->find($identifier);

                if ($symbol === null) {
                    $this->throwException(
                        NotFoundException::class,
                        'Error: Detected unknown or invalid character identifier.',
                        $token->getPosition(),
                        $identifier
                    );
                }

                if (is_a($symbol, AbstractOpeningBracket::class)) {
                    $openBracketCounter++;
                }
                if (is_a($symbol, AbstractClosingBracket::class)) {
                    $openBracketCounter--;

                    // Make sure there are not too many closing brackets
                    if ($openBracketCounter < 0) {
                        $this->throwException(
                            ParserException::class,
                            'Error: Found closing bracket that does not have an opening bracket.',
                            $token->getPosition(),
                            $identifier
                        );
                    }
                }
            }

            // Make sure a function is not followed by a symbol that is not of type opening bracket
            if ($expectingOpeningBracket) {
                if (! is_a($symbol, AbstractOpeningBracket::class)) {
                    $this->throwException(
                        ParserException::class,
                        'Error: Expected opening bracket (after a function) but got something else.',
                        $token->getPosition(),
                        get_class($symbol)
                    );
                }

                $expectingOpeningBracket = false;
            } else {
                if (is_a($symbol, AbstractFunction::class)) {
                    $expectingOpeningBracket = true;
                }
            }

            $symbolNode = new SymbolNode($token, $symbol);

            $symbolNodes[] = $symbolNode;
        }

        // Make sure the term does not end with the name of a function but without an opening bracket
        if ($expectingOpeningBracket) {
            throw new ParserException(
                'Error: Expected opening bracket (after a function) but reached the end of the term'
            );
        }

        // Make sure there are not too many opening brackets
        if ($openBracketCounter > 0) {
            throw new ParserException(
                'Error: There is at least one opening bracket that does not have a closing bracket'
            );
        }

        return $symbolNodes;
    }
792225
public static Token[] optimize(Token[] tokens) {
        if (tokens == null) {
            return null;
        }

        String firstVal = null;
        String lastVal = null;

        if (tokens.length == 1) {
            if (tokens[0].getType() == TokenType.TEXT) {
                firstVal = tokens[0].getDefaultValue();
            }
        } else if (tokens.length > 1) {
            if (tokens[0].getType() == TokenType.TEXT) {
                firstVal = tokens[0].getDefaultValue();
            }
            if (tokens[tokens.length - 1].getType() == TokenType.TEXT) {
                lastVal = tokens[tokens.length - 1].getDefaultValue();
            }
        }

        if (firstVal != null) {
            String text = trimLeadingWhitespace(firstVal);
            if (!Objects.equals(firstVal, text)) {
                tokens[0] = new Token(text);
            }
        }

        if (lastVal != null && !lastVal.isEmpty()) {
            String text = trimTrailingWhitespace(lastVal);
            if (!Objects.equals(lastVal, text)) {
                tokens[tokens.length - 1] = new Token(text);
            }
        }

        return tokens;
    }
364846
public final Span[] nercToSpans(final String[] tokens) {
    final List<Span> neSpans = new ArrayList<Span>();
    for (final Map<String, String> neDict : this.dictionaries
        .getIgnoreCaseDictionaries()) {
      for (final Map.Entry<String, String> neEntry : neDict.entrySet()) {
        final String neForm = neEntry.getKey();
        final String neType = neEntry.getValue();
        final List<Integer> neIds = StringUtils
            .exactTokenFinderIgnoreCase(neForm, tokens);
        if (!neIds.isEmpty()) {
          for (int i = 0; i < neIds.size(); i += 2) {
            final Span neSpan = new Span(neIds.get(i), neIds.get(i + 1),
                neType);
            neSpans.add(neSpan);
            if (this.debug) {
              System.err.println(neSpans.toString());
            }
          }
        }
      }
    }
    return neSpans.toArray(new Span[neSpans.size()]);
  }
570546
protected boolean isTokenPair(AnalyzedTokenReadings[] tokens, int n, boolean before) {
    if (before) {
      if ((tokens[n-2].hasPosTagStartingWith("SUB") && tokens[n-1].hasPosTagStartingWith("PRP")
              && tokens[n].hasPosTagStartingWith("SUB"))
          || (!tokens[n-2].getToken().equals("hart") && !tokens[n-1].getToken().equals("auf") && !tokens[n].getToken().equals("hart"))
         ) {
        return true;
      }
    } else {
      if ((tokens[n].hasPosTagStartingWith("SUB") && tokens[n+1].hasPosTagStartingWith("PRP")
              && tokens[n+2].hasPosTagStartingWith("SUB"))
          || (!tokens[n].getToken().equals("hart") && !tokens[n-1].getToken().equals("auf") && !tokens[n + 2].getToken().equals("hart"))
         ) {
        return true;
      }
    }
    return false;
  }
429962
public List<String> lemmatize(String[] tokens, String[] posTags) {
    String[] annotatedLemmas = lemmatizer.lemmatize(tokens, posTags);
    String[] decodedLemmas = lemmatizer.decodeLemmas(tokens, annotatedLemmas);
    final List<String> lemmas = new ArrayList<String>(Arrays.asList(decodedLemmas));
    return lemmas;
  }