Parse the input string, one token at a time a contruct the call stack
493115
def parse(substatement = nil, token_index = 0)
      p_token = nil

      if substatement
        c_token, c_token_value = substatement[token_index]
      else
        c_token, c_token_value = @scanner.get_token
      end

      parenth = 0

      until c_token.nil?
        if substatement
          token_index += 1
          n_token, n_token_value = substatement[token_index]
        else
          @scanner.token_index += 1
          n_token, n_token_value = @scanner.get_token
        end

        next if n_token == " "
        case c_token

        when "and"
          unless (n_token =~ /not|statement|\(|\+|-/) || (scanner.token_index == scanner.arguments.size)
            raise "Error at column #{scanner.token_index}. \nExpected 'not', 'statement' or '('. Found '#{n_token_value}'"
          end

          raise "Error at column #{scanner.token_index}. \n Expression cannot start with 'and'" if p_token.nil?
          raise "Error at column #{scanner.token_index}. \n #{p_token} cannot be followed by 'and'" if %w[and or].include?(p_token)

        when "or"
          unless (n_token =~ /not|statement|\(|\+|-/) || (scanner.token_index == scanner.arguments.size)
            raise "Error at column #{scanner.token_index}. \nExpected 'not', 'statement', '('. Found '#{n_token_value}'"
          end

          raise "Error at column #{scanner.token_index}. \n Expression cannot start with 'or'" if p_token.nil?
          raise "Error at column #{scanner.token_index}. \n #{p_token} cannot be followed by 'or'" if %w[and or].include?(p_token)

        when "not"
          unless n_token =~ /statement|\(|not|\+|-/
            raise "Error at column #{scanner.token_index}. \nExpected 'statement' or '('. Found '#{n_token_value}'"
          end

        when "statement"
          if c_token_value.is_a? Array
            raise "Error at column #{scanner.token_index}\nError, cannot define '[' in a '[...]' block." if substatement

            parse(c_token_value, 0)
          end

          if c_token_value =~ /!=/
            c_token_value = c_token_value.gsub("!=", "=")
            @execution_stack << {"not" => "not"}
          end

          if !n_token.nil? && !n_token.match(/and|or|\)/)
            raise "Error at column #{scanner.token_index}. \nExpected 'and', 'or', ')'. Found '#{n_token_value}'"
          end

        when "+"
          if !n_token.nil? && !n_token.match(/and|or|\)/)
            raise "Error at column #{scanner.token_index}. \nExpected 'and', 'or', ')'. Found '#{n_token_value}'"
          end

        when "-"
          if !n_token.nil? && !n_token.match(/and|or|\)/)
            raise "Error at column #{scanner.token_index}. \nExpected 'and', 'or', ')'. Found '#{n_token_value}'"
          end

        when ")"
          if !n_token.nil? && !n_token =~ /|and|or|not|\(/
            raise "Error at column #{scanner.token_index}. \nExpected 'and', 'or', 'not' or '('. Found '#{n_token_value}'"
          end

          parenth += 1

        when "("
          unless n_token =~ /statement|not|\(|\+|-/
            raise "Error at column #{scanner.token_index}. \nExpected 'statement', '(',  not. Found '#{n_token_value}'"
          end

          parenth -= 1

        else
          raise "Unexpected token found at column #{scanner.token_index}. '#{c_token_value}'"
        end

        unless n_token == " " || substatement
          @execution_stack << {c_token => c_token_value}
        end

        p_token = c_token
        c_token = n_token
        c_token_value = n_token_value
      end

      return if substatement

      raise "Error. Missing parentheses ')'." if parenth < 0
      raise "Error. Missing parentheses '('." if parenth > 0
    end
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
1084780
def _parse_value(self):
        """"""
        val = []
        while True:
            t = self._next_token()
            if t == '"':
                brac_counter = 0
                while True:
                    t = self._next_token(skipws=False)
                    if t == '{':
                        brac_counter += 1
                    if t == '}':
                        brac_counter -= 1
                    if t == '"' and brac_counter <= 0:
                        break
                    else:
                        val.append(t)
            elif t == '{':
                brac_counter = 0
                while True:
                    t = self._next_token(skipws=False)
                    if t == '{':
                        brac_counter += 1
                    if t == '}':
                        brac_counter -= 1
                    if brac_counter < 0:
                        break
                    else:
                        val.append(t)
            elif re.match(r'\w', t):
                val.extend([self.definitions.get(t, t), ' '])
            elif t.isdigit():
                val.append([t, ' '])
            elif t == '#':
                pass
            else:
                break

        value = ' '.join(''.join(val).split())
        return value
67177
function(str) {
	var tokens = [] // size = str.length + 2
	var i = 0;
	var maxlen = str.length;
	TOKEN:
	while (i < maxlen) {
		var c = str.charAt(i);
		var mlo = m_tokens_start.get(c);
		// if there are multi-char tokens starting with this char, try them
		if (mlo != null) {
			for (var len = mlo; len > 1; len--) {
				if (i <= maxlen - len) {
					var tr = str.substring(i, i + len);
					if (m_tokens.contains(tr)) {
						tokens.push(tr);
						i += len;
						continue TOKEN;
					}
				}
			}
		}
		// things starting with backslash are special
		if (c == '\\' && i <= maxlen - 2) {
			if (str.charAt(i + 1) == 'u' && i <= maxlen - 6) {
				tokens.push(str.substring(i, i + 6));		// \\uxxxx
				i += 6;
			} else if (str.charAt(i + 1) == 'U' && i <= maxlen - 10) {
				tokens.push(str.substring(i, i + 10));		// \\Uxxxxxxxx
				i += 10;
			} else {
				tokens.push(str.substring(i, i + 2));		// \\x
				i += 2;
			}
			continue TOKEN;
		}
		// otherwise just take one char
		tokens.push(c.toString());
		i += 1;
	}
	return tokens;
}
1220231
def parse(cls, string):
		'''
			
		'''
		obj, i = cls.match(string, 0)
		if i != len(string):
			raise Exception("Could not parse '" + string + "' beyond index " + str(i))
		return obj
895742
function TokenStreamBase(input, tokenData){

    /**
     * The string reader for easy access to the text.
     * @type StringReader
     * @property _reader
     * @private
     */
    this._reader = input ? new StringReader(input.toString()) : null;

    /**
     * Token object for the last consumed token.
     * @type Token
     * @property _token
     * @private
     */
    this._token = null;

    /**
     * The array of token information.
     * @type Array
     * @property _tokenData
     * @private
     */
    this._tokenData = tokenData;

    /**
     * Lookahead token buffer.
     * @type Array
     * @property _lt
     * @private
     */
    this._lt = [];

    /**
     * Lookahead token buffer index.
     * @type int
     * @property _ltIndex
     * @private
     */
    this._ltIndex = 0;

    this._ltIndexCache = [];
}
673973
function Parser(str, opts) {
  this._oneWayVoid = !!(opts && opts.oneWayVoid);
  this._reassignJavadoc = !!(opts && opts.reassignJavadoc);
  this._imports = [];
  this._tk = new Tokenizer(str);
  this._tk.next(); // Prime tokenizer.
}
130333
def parse(string)
      @stack = [ ParseContainerTag.new { |t| Utility.array_to_s(t.contents) } ]
      tokenize(string)
      stack_up
      @stack.last.to_s
    end
572574
private static void processToken( String token, Stack<String> stack, Properties props )
    {
        if( "}".equals( token ) )
        {
            String name = stack.pop();
            String open = stack.pop();
            if( "${".equals( open ) )
            {
                startProperty( name, props, stack );
            }
            else
            {
                push( stack, "${" + name + "}" );
            }
        }
        else
        {
            if( "$".equals( token ) )
            {
                stack.push( "$" );
            }
            else
            {
                push( stack, token );
            }
        }
    }
71079
static void parse(String inString, IContext context, INode parent) {
        List<String> tokens = getCommaTokens(inString);
        for (String token : tokens) {
            int isFunction = token.indexOf(OPEN_PARENTHESIS);
            if (isFunction >= 0) {
                String funcName = token.substring(0, isFunction);
                INode newParent = addChild(parent, context, funcName);
                String arguments = token.substring(isFunction + 1, token.length() - 1);
                parse(arguments, context, newParent);
            } else {
                addChild(parent, context, token);
            }
        }
    }
1237768
def call(str)
      result = [:multi]
      reset(str.split(/\r?\n/), [result])

      parse_line while next_line

      reset
      result
    end
1098138
function parseString(str) {
  var obj = {};
  var pairs = str.split('&');
  var pair;
  var pos;

  for (var i = 0, len = pairs.length; i < len; ++i) {
    pair = pairs[i];
    pos = pair.indexOf('=');
    if (pos == -1) {
      obj[decodeURIComponent(pair)] = '';
    } else {
      obj[decodeURIComponent(pair.slice(0, pos))] =
        decodeURIComponent(pair.slice(pos + 1));
    }
  }

  return obj;
}
1250926
function(str) {
            var root, error = null;

            i = j = current = 0;
            chunks = [];
            input = str.replace(/\r\n/g, '\n');
            if (env.filename) {
                that.env.inputs[env.filename] = input;
            }

            // Split the input into chunks.
            chunks = (function (chunks) {
                var j = 0,
                    skip = /(?:@\{[\w-]+\}|[^"'`\{\}\/\(\)\\])+/g,
                    comment = /\/\*(?:[^*]|\*+[^\/*])*\*+\/|\/\/.*/g,
                    string = /"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)'|`((?:[^`]|\\.)*)`/g,
                    level = 0,
                    match,
                    chunk = chunks[0],
                    inParam;

                for (var i = 0, c, cc; i < input.length;) {
                    skip.lastIndex = i;
                    match = skip.exec(input);
                    if (match) {
                        if (match.index === i) {
                            i += match[0].length;
                            chunk.push(match[0]);
                        }
                    }
                    c = input.charAt(i);
                    comment.lastIndex = string.lastIndex = i;

                    match = string.exec(input)
                    if (match) {
                        if (match.index === i) {
                            i += match[0].length;
                            chunk.push(match[0]);
                            continue;
                        }
                    }

                    if (!inParam && c === '/') {
                        cc = input.charAt(i + 1);
                        if (cc === '/' || cc === '*') {
                            match = comment.exec(input);
                            if (match) {
                                if (match.index === i) {
                                    i += match[0].length;
                                    chunk.push(match[0]);
                                    continue;
                                }
                            }
                        }
                    }

                    switch (c) {
                        case '{':
                            if (!inParam) {
                                level ++;
                            }
                            else {
                                inParam = false;
                            }
                            chunk.push(c);
                            break;
                        case '}':
                            if (!inParam) {
                                level --;
                                chunk.push(c);
                                chunks[++j] = chunk = [];
                            }
                            else {
                                inParam = false;
                                chunk.push(c);
                            }
                            break;
                        case '(':
                            if (!inParam) {
                                inParam = true;
                            }
                            else {
                                inParam = false;
                            }
                            chunk.push(c);
                            break;
                        case ')':
                            if (inParam) {
                                inParam = false;
                            }
                            chunk.push(c);
                            break;
                        default:
                            chunk.push(c);
                    }

                    i++;
                }
                if (level !== 0) {
                    error = {
                        index: i - 1,
                        message: (level > 0) ? "missing closing `}`" : "missing opening `{`",
                        filename: env.filename
                    };
                }

                return chunks.map(function (c) { return c.join(''); });
            })([[]]);

            if (error) {
                util.error(env, error);
                throw new Error('N/A');
            }

            // Start with the primary rule.
            // The whole syntax tree is held under a Ruleset node,
            // with the `root` property set to true, so no `{}` are
            // output.
            root = new tree.Ruleset([], $(this.parsers.primary));
            root.root = true;

            // Get an array of Ruleset objects, flattened
            // and sorted according to specificitySort
            root.toList = (function() {
                return function(env) {
                    env.frames = env.frames || [];


                    // call populates Invalid-caused errors
                    var definitions = this.flatten([], [], env);
                    definitions.sort(specificitySort);
                    return definitions;
                };
            })();

            // Sort rules by specificity: this function expects selectors to be
            // split already.
            //
            // Written to be used as a .sort(Function);
            // argument.
            //
            // [1, 0, 0, 467] > [0, 0, 1, 520]
            var specificitySort = function(a, b) {
                var as = a.specificity;
                var bs = b.specificity;

                if (as[0] != bs[0]) return bs[0] - as[0];
                if (as[1] != bs[1]) return bs[1] - as[1];
                if (as[2] != bs[2]) return bs[2] - as[2];
                return bs[3] - as[3];
            };

            return root;
        }
753616
def tokenize(self, string):
        ''''''
        pos = 0
        while pos < len(string):
            m = SKIP_RE.match(string, pos=pos)
            if m:
                skip_lines = m.group(0).split('\n')
                if len(skip_lines) > 1:
                    self.row += len(skip_lines) - 1
                    self.column = 1 + len(skip_lines[-1])
                else:
                    self.column += len(skip_lines[0])

                pos = m.end()
                continue

            for cls, type, regex in self.token_map:
                m = regex.match(string, pos=pos)
                if m:
                    yield cls(type, m.group(0),
                              self.filename, self.row, self.column)
                    self.column += len(m.group(0))
                    pos = m.end()
                    break
            else:
                raise ConfigParseError(
                    "Couldn't load config in %r row %d, column %d: %r" %
                    (self.filename, self.row, self.column,
                     string[pos:pos+20]))
693912
def tokenize(self, string):
        '''
        
        '''

        new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
        new_lexer.latest_newline = 0
        new_lexer.string_value = None
        new_lexer.input(string)

        while True:
            t = new_lexer.token()
            if t is None: break
            t.col = t.lexpos - new_lexer.latest_newline
            yield t

        if new_lexer.string_value is not None:
            raise JsonPathLexerError('Unexpected EOF in string literal or identifier')
1288386
def load(stream):
    """"""
    if isinstance(stream, str):
        string = stream
    else:
        string = stream.read()
    tokens = tokenize(string)
    return parse(tokens)
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())