Example:
            tokens
                dharma.String:      'let ',
                dharma.ElementXRef: 'GrammarNS:<VarName>',
                dharma.String:      '= new ',
                dharma.ValueXRef:   'GrammarNS:<ValueName>'
969973
def parse_assign_variable(self, tokens):
        
        for i, token in enumerate(tokens):
            if isinstance(token, ElementXRef):
                variable = token.value
                break
        else:
            logging.error("%s: Variable assignment syntax error", self.id())
            sys.exit(-1)
        if variable != self.current_obj.ident:
            logging.error("%s: Variable name mismatch", self.id())
            sys.exit(-1)
        if not isinstance(self.current_obj, DharmaVariable):
            logging.error("%s: Inconsistent object for variable assignment", self.id())
            sys.exit(-1)
        prefix, suffix = tokens[:i], tokens[i + 1:]  # pylint: disable=undefined-loop-variable
        self.current_obj.append((prefix, suffix))
238661
private void convertTokenDefinitions(Map<String, ParseTreeNode> helpers,
	    Map<String, ParseTreeNode> tokens) throws GrammarException,
	    TreeException {
	tokenDefinitions = new TokenDefinitionSet();
	for (ParseTreeNode tokenDefinitionAST : parserTree.getChild("Tokens")
		.getChild("TokenDefinitions").getChildren("TokenDefinition")) {
	    TokenDefinition convertedTokenDefinition = getTokenDefinition(
		    tokenDefinitionAST, helpers, tokens);
	    tokenDefinitions.addDefinition(convertedTokenDefinition);
	}
    }
232455
def parse_component_values(input = @tokens)
      input  = TokenScanner.new(input) unless input.is_a?(TokenScanner)
      tokens = []

      while token = consume_component_value(input)
        tokens << token
      end

      tokens
    end
263083
function tokenVariable(stream, state) {
    var isVariableChar = /[\w\$_-]/;

    // a variable may start with a quoted EQName so if the next character is quote, consume to the next quote
    if(stream.eat("\"")) {
      while(stream.next() !== '\"'){};
      stream.eat(":");
    } else {
      stream.eatWhile(isVariableChar);
      if(!stream.match(":=", false)) stream.eat(":");
    }
    stream.eatWhile(isVariableChar);
    state.tokenize = tokenBase;
    return ret("variable", "variable");
  }
625519
def tag(self, tokens):
        """"""
        if not self._loaded_model:
            self.load(self.model)
        tags = [None] * len(tokens)
        norm = self._normalize(tokens)
        length = len(norm)
        # A set of allowed indexes for matches to start or end at
        delims = [0] + [i for span in [m.span() for m in self.delimiters.finditer(norm)] for i in span] + [length]
        # Token indices
        token_at_index = []
        for i, t in enumerate(tokens):
            token_at_index.extend([i] * (len(self.lexicon[t].normalized) + 1))
        start_i = 0
        end_i = 1
        matches = {}
        next_start = end_i
        # TODO: This could be a little more efficient by skipping indexes forward to next delim points.
        while True:
            current = norm[start_i:end_i]
            if self._dawg.has_keys_with_prefix(current):
                # print('%s:%s:%s' % (start_i, end_i, current))
                # If the current span is in the dawg, and isn't followed by an alphanumeric character
                if current in self._dawg and start_i in delims and end_i in delims:
                    # print(current)
                    # Subsequent longer matches with same start_i will overwrite values in matches dict
                    matches[start_i] = (start_i, end_i, current)
                    # We can skip forward to after this match next time we increment start_i
                    next_start = end_i
                # Increment end_i provided we aren't already at the end of the input
                if end_i < length:
                    end_i += 1
                    continue
            # Increment start_i provided we aren't already at the end of the input
            start_i = next_start
            if start_i >= length - 1:
                break
            end_i = start_i + 1
            next_start = end_i
        # Apply matches as tags to the relevant tokens
        for start_i, end_i, current in matches.values():
            start_token = token_at_index[start_i]
            end_token = token_at_index[end_i]
            # Possible for match to start in 'I' token from prev match. Merge matches by not overwriting to 'B'.
            if not tags[start_token] == 'I-%s' % self.entity:
                tags[start_token] = 'B-%s' % self.entity
            tags[start_token+1:end_token+1] = ['I-%s' % self.entity] * (end_token - start_token)
        tokentags = list(zip(tokens, tags))
        return tokentags
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
949431
function varMatchingAny(varnameAtom) {
  return sl.list(sl.atom("::", {token: varnameAtom}), varnameAtom, sl.str("any"));
}
1128883
def visitObjectExpr(self, ctx: jsgParser.ObjectExprContext):
        
        if not self._name:
            self._name = self._context.anon_id()
        if ctx.membersDef():
            self.visitChildren(ctx)
        elif ctx.MAPSTO():
            if ctx.LEXER_ID_REF():
                self._map_name_type = as_token(ctx)
            # Any and absent mean the same thing
            self._map_valuetype = JSGValueType(self._context, ctx.valueType())
            if ctx.ebnfSuffix():
                self._map_ebnf = JSGEbnf(self._context, ctx.ebnfSuffix())
1267407
protected void Literal() throws javax.xml.transform.TransformerException
  {

    int last = m_token.length() - 1;
    char c0 = m_tokenChar;
    char cX = m_token.charAt(last);

    if (((c0 == '\"') && (cX == '\"')) || ((c0 == '\'') && (cX == '\'')))
    {

      // Mutate the token to remove the quotes and have the XString object
      // already made.
      int tokenQueuePos = m_queueMark - 1;

      m_ops.m_tokenQueue.setElementAt(null,tokenQueuePos);

      Object obj = new XString(m_token.substring(1, last));

      m_ops.m_tokenQueue.setElementAt(obj,tokenQueuePos);

      // lit = m_token.substring(1, last);
      m_ops.setOp(m_ops.getOp(OpMap.MAPINDEX_LENGTH), tokenQueuePos);
      m_ops.setOp(OpMap.MAPINDEX_LENGTH, m_ops.getOp(OpMap.MAPINDEX_LENGTH) + 1);

      nextToken();
    }
    else
    {
      error(XPATHErrorResources.ER_PATTERN_LITERAL_NEEDS_BE_QUOTED,
            new Object[]{ m_token });  //"Pattern literal ("+m_token+") needs to be quoted!");
    }
  }
220391
def parse_component_value(input = @tokens)
      input = TokenScanner.new(input) unless input.is_a?(TokenScanner)

      while input.peek && input.peek[:node] == :whitespace
        input.consume
      end

      if input.peek.nil?
        return create_node(:error, :value => 'empty')
      end

      value = consume_component_value(input)

      while input.peek && input.peek[:node] == :whitespace
        input.consume
      end

      if input.peek.nil?
        value
      else
        create_node(:error, :value => 'extra-input')
      end
    end
343989
def parse_tokens(tokens)
    parse_str   = ''
    last_type   = ''
    modifier    = ''
    num_args    = 0
    _class      = 0
    _classary   = [1]
    args        = tokens.dup
    # A note about the class modifiers used below. In general the class,
    # "reference" or "value", of a function is applied to all of its operands.
    # However, in certain circumstances the operands can have mixed classes,
    # e.g. =VLOOKUP with external references. These will eventually be dealt
    # with by the parser. However, as a workaround the class type of a token
    # can be changed via the repeat_formula interface. Thus, a _ref2d token can
    # be changed by the user to _ref2dA or _ref2dR to change its token class.
    #
    while (!args.empty?)
      token = args.shift

      if (token == '_arg')
        num_args = args.shift
      elsif (token == '_class')
        token = args.shift
        _class = @functions[token][2]
        # If _class is undef then it means that the function isn't valid.
        exit "Unknown function #{token}() in formula\n" if _class.nil?
        _classary.push(_class)
      elsif (token == '_vol')
        parse_str += convert_volatile()
      elsif (token == 'ptgBool')
        token = args.shift
        parse_str += convert_bool(token)
      elsif (token == '_num')
        token = args.shift
        parse_str += convert_number(token)
      elsif (token == '_str')
        token = args.shift
        parse_str += convert_string(token)
      elsif (token =~ /^_ref2d/)
        modifier  = token.sub(/_ref2d/, '')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_ref2d(token, _class)
      elsif (token =~ /^_ref3d/)
        modifier  = token.sub(/_ref3d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_ref3d(token, _class)
      elsif (token =~ /^_range2d/)
        modifier  = token.sub(/_range2d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_range2d(token, _class)
      elsif (token =~ /^_range3d/)
        modifier  = token.sub(/_range3d/,'')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_range3d(token, _class)
      elsif (token =~ /^_name/)
        modifier = token.sub(/_name/, '')
        _class      = _classary[-1]
        _class      = 0 if modifier == 'R'
        _class      = 1 if modifier == 'V'
        token      = args.shift
        parse_str += convert_name(token, _class)
      elsif (token == '_func')
        token = args.shift
        parse_str += convert_function(token, num_args.to_i)
        _classary.pop
        num_args = 0 # Reset after use
      elsif @ptg[token]
        parse_str += [@ptg[token]].pack("C")
      else
        # Unrecognised token
        return nil
      end
    end

    parse_str
  end
1297535
public static String asTokenSeparatedValues(String token, Collection<String> strings) {
        StringBuilder newString = new StringBuilder();
        boolean first = true;
        for(String str : strings) {
            if(! first) {
                newString.append(token);
            }
            first = false;
            newString.append(str);
        }
        return newString.toString();
    }
826168
function parseVariable(str, line) {
    var tokens = lexer.read(utils.strip(str)),
      parser,
      out;

    parser = new TokenParser(tokens, filters, escape, line, opts.filename);
    out = parser.parse().join('');

    if (parser.state.length) {
      utils.throwError('Unable to parse "' + str + '"', line, opts.filename);
    }

    /**
     * A parsed variable token.
     * @typedef {object} VarToken
     * @property {function} compile Method for compiling this token.
     */
    return {
      compile: function () {
        return '_output += ' + out + ';\n';
      }
    };
  }
625786
protected TokenList extractTokens(String equation , ManagerTempVariables managerTemp ) {
        // add a space to make sure everything is parsed when its done
        equation += " ";

        TokenList tokens = new TokenList();

        int length = 0;
        boolean again; // process the same character twice
        TokenType type = TokenType.UNKNOWN;
        for( int i = 0; i < equation.length(); i++ ) {
            again = false;
            char c = equation.charAt(i);
            if( type == TokenType.WORD ) {
                if (isLetter(c)) {
                    storage[length++] = c;
                } else {
                    // add the variable/function name to token list
                    String name = new String(storage, 0, length);
                    tokens.add(name);
                    type = TokenType.UNKNOWN;
                    again = true; // process unexpected character a second time
                }
            } else if( type == TokenType.INTEGER ) { // Handle integer numbers.  Until proven to be a float
                if( c == '.' ) {
                    type = TokenType.FLOAT;
                    storage[length++] = c;
                } else if( c == 'e' || c == 'E' ) {
                    type = TokenType.FLOAT_EXP;
                    storage[length++] = c;
                } else if( Character.isDigit(c) ) {
                    storage[length++] = c;
                } else if( isSymbol(c) || Character.isWhitespace(c) ) {
                    int value = Integer.parseInt( new String(storage, 0, length));
                    tokens.add(managerTemp.createInteger(value));
                    type = TokenType.UNKNOWN;
                    again = true; // process unexpected character a second time
                } else {
                    throw new ParseError("Unexpected character at the end of an integer "+c);
                }
            } else if( type == TokenType.FLOAT ) { // Handle floating point numbers
                if( c == '.') {
                    throw new ParseError("Unexpected '.' in a float");
                } else if( c == 'e' || c == 'E' ) {
                    storage[length++] = c;
                    type = TokenType.FLOAT_EXP;
                } else if( Character.isDigit(c) ) {
                    storage[length++] = c;
                } else if( isSymbol(c) || Character.isWhitespace(c) ) {
                    double value = Double.parseDouble( new String(storage, 0, length));
                    tokens.add(managerTemp.createDouble(value));
                    type = TokenType.UNKNOWN;
                    again = true; // process unexpected character a second time
                } else {
                    throw new ParseError("Unexpected character at the end of an float "+c);
                }
            } else if( type == TokenType.FLOAT_EXP ) { // Handle floating point numbers in exponential format
                boolean end = false;
                if( c == '-' ) {
                    char p = storage[length-1];
                    if( p == 'e' || p == 'E') {
                        storage[length++] = c;
                    } else {
                        end = true;
                    }
                } else if( Character.isDigit(c) ) {
                    storage[length++] = c;
                } else if( isSymbol(c) || Character.isWhitespace(c) ) {
                    end = true;
                } else {
                    throw new ParseError("Unexpected character at the end of an float "+c);
                }

                if( end ) {
                    double value = Double.parseDouble( new String(storage, 0, length));
                    tokens.add(managerTemp.createDouble(value));
                    type = TokenType.UNKNOWN;
                    again = true; // process the current character again since it was unexpected
                }
            } else {
                if( isSymbol(c) ) {
                    boolean special = false;
                    if( c == '-' ) {
                        // need to handle minus symbols carefully since it can be part of a number of a minus operator
                        // if next to a number it should be negative sign, unless there is no operator to its left
                        // then its a minus sign.
                        if( i+1 < equation.length() && Character.isDigit(equation.charAt(i+1)) &&
                                (tokens.last == null || isOperatorLR(tokens.last.getSymbol()))) {
                            type = TokenType.INTEGER;
                            storage[0] = c;
                            length = 1;
                            special = true;
                        }
                    }
                    if( !special ) {
                        TokenList.Token t = tokens.add(Symbol.lookup(c));
                        if (t.previous != null && t.previous.getType() == Type.SYMBOL) {
                            // there should only be two symbols in a row if its an element-wise operation
                            if (t.previous.getSymbol() == Symbol.PERIOD) {
                                tokens.remove(t.previous);
                                tokens.remove(t);
                                tokens.add(Symbol.lookupElementWise(c));
                            }
                        }
                    }
                } else if( Character.isWhitespace(c) ) {
                    continue;// ignore white space
                } else {
                    // start adding to the word
                    if( Character.isDigit(c) ) {
                        type = TokenType.INTEGER;
                    } else {
                        type = TokenType.WORD;
                    }
                    storage[0] = c;
                    length = 1;
                }
            }
            // see if it should process the same character again
            if( again )
                i--;
        }

        return tokens;
    }
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
1057190
static Token[] lexx(final String format) {
        final ArrayList<Token> list = new ArrayList<>(format.length());

        boolean inLiteral = false;
        // Although the buffer is stored in a Token, the Tokens are only
        // used internally, so cannot be accessed by other threads
        StringBuilder buffer = null;
        Token previous = null;
        for (int i = 0; i < format.length(); i++) {
            final char ch = format.charAt(i);
            if (inLiteral && ch != '\'') {
                buffer.append(ch); // buffer can't be null if inLiteral is true
                continue;
            }
            Object value = null;
            switch (ch) {
            // TODO: Need to handle escaping of '
            case '\'':
                if (inLiteral) {
                    buffer = null;
                    inLiteral = false;
                } else {
                    buffer = new StringBuilder();
                    list.add(new Token(buffer));
                    inLiteral = true;
                }
                break;
            case 'y':
                value = y;
                break;
            case 'M':
                value = M;
                break;
            case 'd':
                value = d;
                break;
            case 'H':
                value = H;
                break;
            case 'm':
                value = m;
                break;
            case 's':
                value = s;
                break;
            case 'S':
                value = S;
                break;
            default:
                if (buffer == null) {
                    buffer = new StringBuilder();
                    list.add(new Token(buffer));
                }
                buffer.append(ch);
            }

            if (value != null) {
                if (previous != null && previous.getValue().equals(value)) {
                    previous.increment();
                } else {
                    final Token token = new Token(value);
                    list.add(token);
                    previous = token;
                }
                buffer = null;
            }
        }
        if (inLiteral) { // i.e. we have not found the end of the literal
            throw new IllegalArgumentException("Unmatched quote in format: " + format);
        }
        return list.toArray(new Token[list.size()]);
    }
240268
@Override
	public List<Token> tokenize(String s) {
		List<Token> result = new ArrayList<Token>();
		result.add(new UndefinedToken(s));

		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
			for (Pattern pattern : patterns) {
				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
					Token token = it.next();
					if (token instanceof UndefinedToken) {
						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
								pattern);
						if (replacementTokens.size() > 1) {
							it.remove();
							for (Token newToken : replacementTokens) {
								it.add(newToken);
							}
						}
					}
				}
			}
		}

		return result;
	}