This method converts the token definitions to the token definition set.

@throws GrammarException
@throws TreeException
544560
private void convertTokenDefinitions() throws GrammarException,
	    TreeException {
	tokenVisibility.clear();
	Map<String, ParseTreeNode> helpers = getHelperTokens();
	Map<String, ParseTreeNode> tokens = getTokens();
	convertTokenDefinitions(helpers, tokens);
    }
831618
void reloadDefinitions(
			Map<String, ? extends Object> keywordDefinitions, 
			Map<Integer, ? extends Object> lengthDefinitions){
			this.lengthDefinitions = lengthDefinitions == null ?
					null : new TreeMap<Integer, Object>(lengthDefinitions);
	}
1204305
@Override
    protected List<Token> initializeTokens( List<Token> tokens ) {
        // THIS IS WHERE WE DO THE WORK OF PRE-PARSING TOKENS AND REPLACING KEYWORDS AND STATEMENT STARTS WITH
        // APPLICABLE TOKEN TYPE BITMASK VALUES
        // MyClass[] array = (MyClass[])list.toArray(new MyClass[list.size()]);

        Token[] tokensArray = tokens.toArray(new Token[tokens.size()]);
        List<Token> reTypedTokens = new ArrayList<Token>(tokens.size());

        for (int i = 0; i < tokensArray.length; i++) {
            boolean isStatementStart = false;
            if (isKeyWord(tokensArray[i].value())) {
                Token retypedToken = tokensArray[i].withType(DdlTokenizer.KEYWORD);
                // Now we check to see if this keyword begins a registered statement start

                // Keep track of token increment (# of tokens for a phrase)
                // Need to increment iterator (i) in case phrases like "ALTER ROLLBACK" appear. ROLLBACK is also a statement
                // start phrase and we need to walk ignore ROLLBACK in this case.
                int tokenIncrement = 0;
                for (String[] nextStmtStart : registeredStatementStartPhrases) {
                    boolean matches = true;

                    for (int j = 0; j < nextStmtStart.length; j++) {
                        if (matches) {
                            matches = nextStmtStart[j].equalsIgnoreCase(tokensArray[i + j].value())
                                      || nextStmtStart[j].equals(ANY_VALUE);
                        }
                    }
                    if (matches) {
                        isStatementStart = true;
                        tokenIncrement = nextStmtStart.length - 1;
                        break;
                    }
                }
                if (isStatementStart) {
                    retypedToken = retypedToken.withType(DdlTokenizer.STATEMENT_KEY);
                }
                reTypedTokens.add(retypedToken);

                if (isStatementStart) {
                    // Copy any additional tokens used in the phrase
                    for (int k = 0; k < tokenIncrement; k++) {
                        i++;
                        reTypedTokens.add(tokensArray[i]);
                    }
                }
            } else {
                reTypedTokens.add(tokensArray[i]);
            }

        }

        return reTypedTokens;
    }
275899
public final void literal() throws RecognitionException {
		CommonTree theliteral=null;

		try {
			// src/main/resources/org/drools/compiler/lang/dsl/DSLMapWalker.g:123:5: (theliteral= VT_LITERAL )
			// src/main/resources/org/drools/compiler/lang/dsl/DSLMapWalker.g:123:7: theliteral= VT_LITERAL
			{
			theliteral=(CommonTree)match(input,VT_LITERAL,FOLLOW_VT_LITERAL_in_literal403); 
			}

		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
		}
	}
1013198
private function doTransitionFromD() : self
    {
        if ($this->getToken() === Grammar::T_L) {
            $this
                ->setState(self::STATE_C)
                ->addPosition(1)
                ->addTokenValue(Grammar::T_L);
            return $this;
        }

        if ($this->getToken() === Grammar::T_X && $this->hasToken(1) && $this->getToken(1) === Grammar::T_L) {
            $this
                ->setState(self::STATE_B)
                ->addPosition(2)
                ->addTokenValue(Grammar::T_L, Grammar::T_X);
            return $this;
        }

        if ($this->getToken() === Grammar::T_X && $this->hasToken(1) && $this->getToken(1) === Grammar::T_C) {
            $this
                ->setState(self::STATE_B)
                ->addPosition(2)
                ->addTokenValue(Grammar::T_C, Grammar::T_X);
            return $this;
        }

        $this->setState(self::STATE_C);
        return $this;
    }
211569
public void process(JCas jcas) throws AnalysisEngineProcessException {
		// if the annotate_tokens flag is set, annotate the tokens and add them to the jcas
		if(annotate_tokens)
			if(language.equals(Language.CHINESE))
				tokenizeChinese(jcas); // chinese needs different tokenization
			else
				tokenize(jcas);

		/* if the annotate_partofspeech flag is set, annotate partofspeech and,
		 * if specified, also tag sentences based upon the partofspeech tags. 
		 */
		if(annotate_partofspeech) 
			doTreeTag(jcas);
		
		// if the improve_german_sentences flag is set, improve the sentence tokens made by the treetagger
		if(this.language == Language.GERMAN)
			improveGermanSentences(jcas);
		
		// if French, improve the sentence tokens made by the TreeTagger with settings for French
		if (this.language == Language.FRENCH)
			improveFrenchSentences(jcas);
		
	}
647399
private static Map<Class<? extends CoreAnnotation>, Set<TreeGraphNode>> getAllDependents(TreeGraphNode node) {
    Map<Class<? extends CoreAnnotation>, Set<TreeGraphNode>> newMap = Generics.newHashMap();

    for (Class<?> o : node.label.keySet()) {
      try {
        // The line below will exception unless it's a GrammaticalRelationAnnotation,
        // so the effect is that only the GrammaticalRelationAnnotation things get put into newMap
        o.asSubclass(GrammaticalRelationAnnotation.class);
        newMap.put((Class<? extends CoreAnnotation>) o, (Set<TreeGraphNode>) node.label.get((Class<? extends CoreAnnotation>) o));//javac doesn't compile properly if generics are fully specified (but eclipse does...)
      } catch (Exception e) {
        // ignore a non-GrammaticalRelationAnnotation element
      }
    }
    return newMap;
  }
1204264
private function doTransitionFromE() : self
    {
        if ($this->getToken() === Grammar::T_C) {
            if ($this->hasToken(1) && $this->getToken(1) === Grammar::T_C) {
                if ($this->hasToken(2) && $this->getToken(2) === Grammar::T_C) {
                    $this
                        ->setState(self::STATE_D)
                        ->addPosition(3)
                        ->addTokenValue(Grammar::T_C, null, 3);
                    return $this;
                }

                $this
                    ->setState(self::STATE_D)
                    ->addPosition(2)
                    ->addTokenValue(Grammar::T_C, null, 2);
                return $this;
            }

            $this
                ->setState(self::STATE_D)
                ->addPosition(1)
                ->addTokenValue(Grammar::T_C);
            return $this;
        }

        $this->setState(self::STATE_D);
        return $this;
    }
745096
public function consumeClosingParenthesisToken(): void
    {
        if ($this->currentAnnotationContent !== null && $this->openedStringToken === null) {
            // We are in an annotation content and not in a string.
            if ($this->openedParenthesis > 0) {
                $this->openedParenthesis--;
                if ($this->openedParenthesis === 0) {
                    // Annotation content is finished.
                    $this->currentAnnotation->setStringContent(substr(
                        $this->currentAnnotationContent,
                        1,
                        strlen($this->currentAnnotationContent) - 2
                    ));
                    $this->parsedAnnotations[] = $this->currentAnnotation;
                    $this->reset();
                }
            }
        }
    }
1150630
private function doTransitionFromA() : self
    {
        if ($this->getToken() === Grammar::T_I) {
            if ($this->hasToken(1) && $this->getToken(1) === Grammar::T_I) {
                if ($this->hasToken(2) && $this->getToken(2) === Grammar::T_I) {
                    $this
                        ->setState(self::STATE_Y)
                        ->addPosition(3)
                        ->addTokenValue(Grammar::T_I, null, 3);
                    return $this;
                }

                $this
                    ->setState(self::STATE_Y)
                    ->addPosition(2)
                    ->addTokenValue(Grammar::T_I, null, 2);
                return $this;
            }

            $this
                ->setState(self::STATE_Y)
                ->addPosition(1)
                ->addTokenValue(Grammar::T_I);
            return $this;
        }

        $this->setState(self::STATE_Y);
        return $this;
    }
1128089
public final void mELEMENTS() throws RecognitionException {
		try {
			int _type = ELEMENTS;
			int _channel = DEFAULT_TOKEN_CHANNEL;
			// hql.g:21:10: ( 'elements' )
			// hql.g:21:12: 'elements'
			{
			match("elements"); if (state.failed) return;

			}

			state.type = _type;
			state.channel = _channel;
		}
		finally {
			// do for sure before leaving
		}
	}
505883
private void parseParenthesizedExpr() throws TTXPathException {

        consume(TokenType.OPEN_BR, true);
        if (!(mToken.getType() == TokenType.CLOSE_BR)) {
            parseExpression();
        }
        consume(TokenType.CLOSE_BR, true);

    }
1046960
protected AbstractTokenizer generateTokenizer() {
        Class<? extends AbstractTokenizer> tokenizer = parameters.getTokenizer();
        if(tokenizer==null) {
            return null;
        }

        try {
            return tokenizer.newInstance();
        } 
        catch (InstantiationException | IllegalAccessException ex) {
            throw new RuntimeException(ex);
        }
    }
93282
def parse(self, tokenized):  # pylint: disable=invalid-name
        """"""
        table, trees = _parse(tokenized, self.grammar)
        # Check if the parse succeeded.
        if all(r.lhs != self.start for r in table[(0, len(tokenized) - 1)]):
            raise ParseError('Parsing failed.')
        parse = trees[(0, len(tokenized) - 1)][self.start]
        return self._to_tree(revert_cnf(parse))
633208
protected void parseNodeTypeDefinition( TokenStream tokens ) {
        // Parse the name, and create the path and a property for the name ...
        Name name = parseNodeTypeName(tokens);
        JcrNodeTypeTemplate nodeType = new JcrNodeTypeTemplate(context);
        try {
            nodeType.setName(string(name));
        } catch (ConstraintViolationException e) {
            assert false : "Names should always be syntactically valid";
        }

        // Read the (optional) supertypes ...
        List<Name> supertypes = parseSupertypes(tokens);
        try {
            nodeType.setDeclaredSuperTypeNames(names(supertypes));

            // Read the node type options (and vendor extensions) ...
            parseNodeTypeOptions(tokens, nodeType);

            // Parse property and child node definitions ...
            parsePropertyOrChildNodeDefinitions(tokens, nodeType);
        } catch (ConstraintViolationException e) {
            assert false : "Names should always be syntactically valid";
        }

        this.nodeTypes.add(nodeType);
    }
1120282
private final ParseTreeNode parse() throws ParserException {
	try {
	    createActionStack();
	    return LRTokenStreamConverter.convert(getTokenStream(), getGrammar(), actionStack);
	} catch (GrammarException e) {
	    logger.error(e.getMessage(), e);
	    throw new ParserException(e.getMessage());
	}
    }
244951
private void read() throws IOException, GrammarException {
	try {
	    logger.debug("Read grammar file...");
	    logger.debug("Starting lexer...");
	    Lexer lexer = new RegExpLexer(uhuraGrammar);
	    TokenStream tokenStream = lexer.lex(SourceCode.read(reader,
		    new UnspecifiedSourceCodeLocation()));
	    logger.debug("Starting parser...");
	    parse(tokenStream);
	    logger.debug("done reading grammar file.");
	} catch (LexerException e) {
	    logger.error(e.getMessage(), e);
	    throw new IOException(e.getMessage(), e);
	} catch (ParserException e) {
	    logger.error(e.getMessage(), e);
	    throw new IOException(e.getMessage(), e);
	}
    }