Parse the content of a $-string using a lexer

    The lexer is chosen looking for a nearby LANGUAGE or assumed as
    plpgsql if inside a DO statement and no LANGUAGE has been found.
197365
def language_callback(lexer, match):
    
    l = None
    m = language_re.match(lexer.text[match.end():match.end()+100])
    if m is not None:
        l = lexer._get_lexer(m.group(1))
    else:
        m = list(language_re.finditer(
            lexer.text[max(0, match.start()-100):match.start()]))
        if m:
            l = lexer._get_lexer(m[-1].group(1))
        else:
            m = list(do_re.finditer(
                lexer.text[max(0, match.start()-25):match.start()]))
            if m:
                l = lexer._get_lexer('plpgsql')
    
    # 1 = $, 2 = delimiter, 3 = $
    yield (match.start(1), String, match.group(1))
    yield (match.start(2), String.Delimiter, match.group(2))
    yield (match.start(3), String, match.group(3))
    # 4 = string contents
    if l:
        for x in l.get_tokens_unprocessed(match.group(4)):
            yield x
    else:
        yield (match.start(4), String, match.group(4))
    # 5 = $, 6 = delimiter, 7 = $
    yield (match.start(5), String, match.group(5))
    yield (match.start(6), String.Delimiter, match.group(6))
    yield (match.start(7), String, match.group(7))
547468
function parseSearchString(str) {

    str = tia.cU.replaceXTypesInTeq(str);

    var re = /&(\w|\d|_|-)+/g;

    var searchData = [];
    var prevLastIndex = 0;
    var query;

    while (true) {
      var reResult = re.exec(str);

      if (reResult === null) {
        // Only query string.
        query = str.slice(prevLastIndex).trim();
        if (query) {
          searchData.push({
            query: query,
          });
        }
        return searchData;
      }

      query = str.slice(prevLastIndex, reResult.index).trim();
      var reference = str.slice(reResult.index + 1, re.lastIndex).trim();
      prevLastIndex = re.lastIndex;

      if (query) {
        searchData.push({
          query: query,
        });
      }

      searchData.push({
        reference: reference,
      });

    }
  }
6943
function parse (string) {
  if (!string || typeof string !== 'string') {
    throw new TypeError('argument string is required')
  }

  var match = DISPOSITION_TYPE_REGEXP.exec(string)

  if (!match) {
    throw new TypeError('invalid type format')
  }

  // normalize type
  var index = match[0].length
  var type = match[1].toLowerCase()

  var key
  var names = []
  var params = {}
  var value

  // calculate index to start at
  index = PARAM_REGEXP.lastIndex = match[0].substr(-1) === ';'
    ? index - 1
    : index

  // match parameters
  while ((match = PARAM_REGEXP.exec(string))) {
    if (match.index !== index) {
      throw new TypeError('invalid parameter format')
    }

    index += match[0].length
    key = match[1].toLowerCase()
    value = match[2]

    if (names.indexOf(key) !== -1) {
      throw new TypeError('invalid duplicate parameter')
    }

    names.push(key)

    if (key.indexOf('*') + 1 === key.length) {
      // decode extended value
      key = key.slice(0, -1)
      value = decodefield(value)

      // overwrite existing value
      params[key] = value
      continue
    }

    if (typeof params[key] === 'string') {
      continue
    }

    if (value[0] === '"') {
      // remove quotes and escapes
      value = value
        .substr(1, value.length - 2)
        .replace(QESC_REGEXP, '$1')
    }

    params[key] = value
  }

  if (index !== -1 && index !== string.length) {
    throw new TypeError('invalid parameter format')
  }

  return new ContentDisposition(type, params)
}
513641
private function lexExpression()
    {
        $statementMethods = $this->getStatementLexingMethods();

        while (!$this->atEof()) {
            reset($statementMethods);
            $matchedStatement = false;

            // This is essentially a foreach loop that can be reset
            while (list($statementOpenDelimiter, $methodName) = each($statementMethods)) {
                if ($this->matches($statementOpenDelimiter)) {
                    // This is an unescaped statement
                    $matchedStatement = true;
                    $this->{$methodName}();

                    // Now that we've matched, we want to reset the loop so that longest delimiters are matched first
                    reset($statementMethods);
                } elseif ($this->getCurrentChar() === '\\') {
                    // Now that we know we're on an escape character, spend the resources to check for a match
                    if ($this->matches("\\$statementOpenDelimiter")) {
                        // This is an escaped statement
                        $this->expressionBuffer .= $statementOpenDelimiter;
                    }
                }
            }

            // Handle any text outside statements
            if (!$matchedStatement && !$this->atEof()) {
                $currentChar = $this->getCurrentChar();
                $this->expressionBuffer .= $currentChar;
                $this->cursor++;

                // Keep on going if we're seeing alphanumeric text
                while (ctype_alnum($this->getCurrentChar())) {
                    $currentChar = $this->getCurrentChar();
                    $this->expressionBuffer .= $currentChar;
                    $this->cursor++;
                }
            } else {
                $this->flushExpressionBuffer();
            }
        }

        // Flush anything left over in the buffer
        $this->flushExpressionBuffer();
    }
947857
def parse(scanner, rules, scope = ParserScope.empty)
      rules.inherited_rule(rule_name).parse(scanner, rules, scope)
    end
1212795
function parseQueryString(query) {
  const result = {};

  _.each(query.replace(/\+/g, ' ').split('&'), (key) => {
    const index = key.indexOf('=');
    let val = '';

    if (index >= 0) {
      val = decodeURIComponent(key.slice(index + 1));
      key = key.slice(0, index);
    }
    if (!key) return;

    key = decodeURIComponent(key);

    // brackets notation for array value
    if (key.slice(-2) === '[]') {
      key = key.slice(0, -2);
      val = [val];
    }

    // if there are multiple values per key, concatenate
    result[key] = _.has(result, key) ? [].concat(result[key], val) : val;
  });

  return result;
}
336642
def _lex(s):
    
    s += '.'  # make sure there's a terminator to know when to stop parsing
    lines = enumerate(s.splitlines(), 1)
    lineno = pos = 0
    try:
        for lineno, line in lines:
            matches = _tsql_lex_re.finditer(line)
            for m in matches:
                gid = m.lastindex
                if gid == 11:
                    raise TSQLSyntaxError('unexpected input',
                                          lineno=lineno,
                                          offset=m.start(),
                                          text=line)
                else:
                    token = m.group(gid)
                    yield (gid, token, lineno)
    except StopIteration:
        pass
567794
private Map parseObject(JsonLexer lexer) {
        Map content = new HashMap();

        JsonToken previousToken = null;
        JsonToken currentToken = null;

        for(;;) {
            currentToken = lexer.nextToken();

            if (currentToken == null) {
                throw new JsonException(
                        "Expected a String key on line: " + lexer.getReader().getLine() + ", " +
                                "column: " + lexer.getReader().getColumn() + ".\n" +
                                "But got an unterminated object."
                );
            }

            // expect a string key, or already a closing curly brace

            if (currentToken.getType() == CLOSE_CURLY) {
                return content;
            } else if (currentToken.getType() != STRING) {
                throw new JsonException(
                        "Expected " + STRING.getLabel() + " key " +
                                "on line: " + currentToken.getStartLine() + ", " +
                                "column: " + currentToken.getStartColumn() + ".\n" +
                                "But got '" + currentToken.getText() + "' instead."
                );
            }

            String mapKey = (String) currentToken.getValue();

            currentToken = lexer.nextToken();

            if (currentToken == null) {
                throw new JsonException(
                        "Expected a " + COLON.getLabel() + " " +
                                "on line: " + lexer.getReader().getLine() + ", " +
                                "column: " + lexer.getReader().getColumn() + ".\n" +
                                "But got an unterminated object."
                );
            }

            // expect a colon between the key and value pair

            if (currentToken.getType() != COLON) {
                throw new JsonException(
                        "Expected " + COLON.getLabel() + " " +
                                "on line: " + currentToken.getStartLine() + ", " +
                                "column: " + currentToken.getStartColumn() + ".\n" +
                                "But got '" + currentToken.getText() + "' instead."
                );
            }

            currentToken = lexer.nextToken();

            if (currentToken == null) {
                throw new JsonException(
                        "Expected a value " +
                                "on line: " + lexer.getReader().getLine() + ", " +
                                "column: " + lexer.getReader().getColumn() + ".\n" +
                                "But got an unterminated object."
                );
            }

            // value can be an object, an array, a number, string, boolean or null values

            if (currentToken.getType() == OPEN_CURLY) {
                content.put(mapKey, parseObject(lexer));
            } else if (currentToken.getType() == OPEN_BRACKET) {
                content.put(mapKey, parseArray(lexer));
            } else if (currentToken.getType().ordinal() >= NULL.ordinal()) {
                content.put(mapKey, currentToken.getValue());
            } else {
                throw new JsonException(
                        "Expected a value, an array, or an object " +
                                "on line: " + currentToken.getStartLine() + ", " +
                                "column: " + currentToken.getStartColumn() + ".\n" +
                                "But got '" + currentToken.getText() + "' instead."
                );
            }

            previousToken = currentToken;
            currentToken = lexer.nextToken();

            // premature end of the object

            if (currentToken == null) {
                throw new JsonException(
                        "Expected " + CLOSE_CURLY.getLabel() + " or " + COMMA.getLabel() + " " +
                                "on line: " + previousToken.getEndLine() + ", " +
                                "column: " + previousToken.getEndColumn() + ".\n" +
                                "But got an unterminated object."
                );
            }

            // Expect a comma for an upcoming key/value pair
            // or a closing curly brace for the end of the object
            if (currentToken.getType() == CLOSE_CURLY) {
                break;
            } else if (currentToken.getType() != COMMA) {
                throw new JsonException(
                        "Expected a value or " + CLOSE_CURLY.getLabel() + " " +
                                "on line: " + currentToken.getStartLine() + ", " +
                                "column: " + currentToken.getStartColumn() + ".\n" +
                                "But got '" + currentToken.getText() + "' instead."
                );
            }
        }

        return content;
    }
1141634
public function parse()
    {
        $this->lexer->moveNext();

        while (true) {
            if (!$this->lexer->lookahead) {
                break;
            }

            $this->lexer->moveNext();
            $thisLine = null;

            switch ($this->lexer->token['type']) {
                case Lexer::T_OPEN_PARENTHESIS:
                case Lexer::T_OPEN_BRACKET:
                case Lexer::T_OPEN_CURLY_BRACE:
                    $thisLine = '[';
                    $this->currentLevel++;
                    break;
                case Lexer::T_CLOSE_PARENTHESIS:
                case Lexer::T_CLOSE_BRACKET:
                case Lexer::T_CLOSE_CURLY_BRACE:
                    $thisLine = ']';
                    $this->currentLevel--;
                    break;
                case Lexer::T_STRING:
                    $thisLine = "'".str_replace("'", "\'", $this->lexer->token['value'])."'";
                    break;
                case Lexer::T_DOUBLEPOINT:
                    $thisLine = '=>';
                    break;
                case Lexer::T_COMMENT_PARAMSTART:
                    $paramName = $this->lexer->lookahead['value'];
                    $thisLine = '$this->getParam("'.$paramName.'")';
                    // scroll to end of param
                    while ($this->lexer->token['type'] != Lexer::T_COMMENT_PARAMEND) {
                        $this->lexer->moveNext();
                    }
                    break;
                case Lexer::T_COMMENT_IFPARAMSTART:
                case Lexer::T_COMMENT_IFNOTPARAMSTART:
                    $conditionalId = str_pad((string) $this->conditionalCounter, 5, '0', STR_PAD_LEFT);
                    $this->conditionalCounter++;

                    $isNegated = '';
                    if ($this->lexer->token['type'] == Lexer::T_COMMENT_IFNOTPARAMSTART) {
                        $isNegated = 'true';
                    }

                    $this->addContent('$this->getConditional'.$conditionalId.'('.$isNegated.'),');
                    $this->openConditional($conditionalId, $this->lexer->lookahead['value']);
                    break;
                case Lexer::T_COMMENT_IFPARAMEND:
                case Lexer::T_COMMENT_IFNOTPARAMEND:
                    $this->closeConditional();
                    break;
                case Lexer::T_DATE_EMPTY:
                    $thisLine = 'new '.$this->mongoDateClass.'()';
                    break;
                case Lexer::T_DATE_WITH_PARAM:
                    $paramName = $this->lexer->lookahead['value'];
                    $thisLine = '$this->getParam("'.$paramName.'")';
                    break;
                case Lexer::T_BOOLEAN_TRUE:
                    $thisLine = 'true';
                    break;
                case Lexer::T_BOOLEAN_FALSE:
                    $thisLine = 'false';
                    break;
                case Lexer::T_NULL:
                    $thisLine = 'null';
                    break;
                case Lexer::T_COMMA:
                case Lexer::T_NUMBER:
                case Lexer::T_DASH:
                    $thisLine = $this->lexer->token['value'];
                    break;
                default:
                    // nothing
            }

            if (!is_null($thisLine)) {
                $this->addContent($thisLine);
            }
        }

        return implode(PHP_EOL, $this->lines);
    }
528623
def _parse(cls, scope):
    
    if not scope:
      return ('default',)
    if isinstance(scope, string_types):
      scope = scope.split(' ')
    scope = {str(s).lower() for s in scope if s}
    return scope or ('default',)
912402
public boolean parseString(final String pStringToParse) {

        if (debugging) {
            out.println("parsing \"" + pStringToParse + "\"...");
        }

        // Update statistics
        totalNumberOfStringsParsed++;

        // Check string to be parsed for nullness
        if (pStringToParse == null) {
            if (debugging) {
                out.println("string to be parsed is null - rejection!");
            }
            return false;
        }

        // Create parsable string
        ParsableString parsableString = new ParsableString(pStringToParse);

        // Check string to be parsed
        if (!parsableString.checkString()) {
            if (debugging) {
                out.println("one or more characters in string to be parsed are not legal characters - rejection!");
            }
            return false;
        }

        // Check if automaton is correctly initialized
        if (!initialized) {
            System.err.println("automaton is not initialized - rejection!");
            return false;
        }

        // Check if automaton is trivial (accepts all strings)
        if (isTrivialAutomaton()) {
            if (debugging) {
                out.println("automaton represents a trivial string mask (accepts all strings) - acceptance!");
            }
            return true;
        }

        // Check if string to be parsed is empty
        if (parsableString.isEmpty()) {
            if (debugging) {
                out.println("string to be parsed is empty and not trivial automaton - rejection!");
            }
            return false;
        }

        // Flag and more to indicate that state skipping due to sequence of '?' succeeding a '*' has been performed
        boolean hasPerformedFreeRangeMovement = false;
        int numberOfFreePassCharactersRead_SinceLastFreePassState = 0;
        int numberOfParsedCharactersRead_SinceLastFreePassState = 0;
        WildcardStringParserState runnerState = null;

        // Accepted by the first state?
        if ((parsableString.charArray[0] == initialState.character) || isWildcardCharacter(initialState.character)) {
            runnerState = initialState;
            parsableString.index = 0;
        }
        else {
            if (debugging) {
                out.println("cannot enter first automaton state - rejection!");
            }
            return false;
        }

        // Initialize the free-pass character state visited count
        if (isFreePassCharacter(runnerState.character)) {
            numberOfFreePassCharactersRead_SinceLastFreePassState++;
        }

        // Perform parsing according to the rules above
        for (int i = 0; i < parsableString.length(); i++) {
            if (debugging) {
                out.println();
            }
            if (debugging) {
                out.println("parsing - index number " + i + ", active char: '"
                        + parsableString.getActiveChar() + "' char string index: " + parsableString.index
                        + " number of chars since last free-range state: " + numberOfParsedCharactersRead_SinceLastFreePassState);
            }
            if (debugging) {
                out.println("parsing - state: " + runnerState.automatonStateNumber + " '"
                        + runnerState.character + "' - no of free-pass chars read: " + numberOfFreePassCharactersRead_SinceLastFreePassState);
            }
            if (debugging) {
                out.println("parsing - hasPerformedFreeRangeMovement: " + hasPerformedFreeRangeMovement);
            }
            if (runnerState.nextState == null) {
                if (debugging) {
                    out.println("parsing - runnerState.nextState == null");
                }

                // If there are no subsequent state (final state) and the state represents '*' - acceptance!
                if (isFreeRangeCharacter(runnerState.character)) {

                    // Special free-range skipping check
                    if (hasPerformedFreeRangeMovement) {
                        if (parsableString.reachedEndOfString()) {
                            if (numberOfFreePassCharactersRead_SinceLastFreePassState > numberOfParsedCharactersRead_SinceLastFreePassState) {
                                if (debugging) {
                                    out.println(
                                            "no subsequent state (final state) and the state represents '*' - end of parsing string, but not enough characters read - rejection!");
                                }
                                return false;
                            }
                            else {
                                if (debugging) {
                                    out.println(
                                            "no subsequent state (final state) and the state represents '*' - end of parsing string and enough characters read - acceptance!");
                                }
                                return true;
                            }
                        }
                        else {
                            if (numberOfFreePassCharactersRead_SinceLastFreePassState > numberOfParsedCharactersRead_SinceLastFreePassState) {
                                if (debugging) {
                                    out.println(
                                            "no subsequent state (final state) and the state represents '*' - not the end of parsing string and not enough characters read - read next character");
                                }
                                parsableString.index++;
                                numberOfParsedCharactersRead_SinceLastFreePassState++;
                            }
                            else {
                                if (debugging) {
                                    out.println(
                                            "no subsequent state (final state) and the state represents '*' - not the end of parsing string, but enough characters read - acceptance!");
                                }
                                return true;
                            }
                        }
                    }
                    else {
                        if (debugging) {
                            out.println("no subsequent state (final state) and the state represents '*' - no skipping performed - acceptance!");
                        }
                        return true;
                    }
                }

                // If there are no subsequent state (final state) and no skipping has been performed and the end of the string to test is reached - acceptance!
                else if (parsableString.reachedEndOfString()) {

                    // Special free-range skipping check
                    if ((hasPerformedFreeRangeMovement)
                            && (numberOfFreePassCharactersRead_SinceLastFreePassState > numberOfParsedCharactersRead_SinceLastFreePassState)) {
                        if (debugging) {
                            out.println(
                                    "no subsequent state (final state) and skipping has been performed and end of parsing string, but not enough characters read - rejection!");
                        }
                        return false;
                    }
                    if (debugging) {
                        out.println("no subsequent state (final state) and the end of the string to test is reached - acceptance!");
                    }
                    return true;
                }
                else {
                    if (debugging) {
                        out.println("parsing - escaping process...");
                    }
                }
            }
            else {
                if (debugging) {
                    out.println("parsing - runnerState.nextState != null");
                }

                // Special Case:
                // If this state represents '*' - go to the rightmost state representing '?'.
                //    This state will act as an '*' - except that you only can go to the next state or accept the string, if and only if the number of '?' read are equal or less than the number of character read from the parsing string.
                if (isFreeRangeCharacter(runnerState.character)) {
                    numberOfFreePassCharactersRead_SinceLastFreePassState = 0;
                    numberOfParsedCharactersRead_SinceLastFreePassState = 0;
                    WildcardStringParserState freeRangeRunnerState = runnerState.nextState;

                    while ((freeRangeRunnerState != null) && (isFreePassCharacter(freeRangeRunnerState.character))) {
                        runnerState = freeRangeRunnerState;
                        hasPerformedFreeRangeMovement = true;
                        numberOfFreePassCharactersRead_SinceLastFreePassState++;
                        freeRangeRunnerState = freeRangeRunnerState.nextState;
                    }

                    // Special Case: if the mask is at the end
                    if (runnerState.nextState == null) {
                        if (debugging) {
                            out.println();
                        }
                        if (debugging) {
                            out.println("parsing - index number " + i + ", active char: '"
                                    + parsableString.getActiveChar() + "' char string index: " + parsableString.index
                                    + " number of chars since last free-range state: " + numberOfParsedCharactersRead_SinceLastFreePassState);
                        }
                        if (debugging) {
                            out.println("parsing - state: " + runnerState.automatonStateNumber + " '"
                                    + runnerState.character + "' - no of free-pass chars read: " + numberOfFreePassCharactersRead_SinceLastFreePassState);
                        }
                        if (debugging) {
                            out.println("parsing - hasPerformedFreeRangeMovement: "
                                    + hasPerformedFreeRangeMovement);
                        }
                        if ((hasPerformedFreeRangeMovement)
                                && (numberOfFreePassCharactersRead_SinceLastFreePassState >= numberOfParsedCharactersRead_SinceLastFreePassState)) {
                            return true;
                        }
                        else {
                            return false;
                        }
                    }
                }

                // If the next state represents '*' - go to this next state
                if (isFreeRangeCharacter(runnerState.nextState.character)) {
                    runnerState = runnerState.nextState;
                    parsableString.index++;
                    numberOfParsedCharactersRead_SinceLastFreePassState++;
                }

                // If the next state represents '?' - go to this next state
                else if (isFreePassCharacter(runnerState.nextState.character)) {
                    runnerState = runnerState.nextState;
                    parsableString.index++;
                    numberOfFreePassCharactersRead_SinceLastFreePassState++;
                    numberOfParsedCharactersRead_SinceLastFreePassState++;
                }

                // If the next state represents the same character as the next character in the string to test - go to this next state
                else if ((!parsableString.reachedEndOfString()) && (runnerState.nextState.character == parsableString.getSubsequentChar())) {
                    runnerState = runnerState.nextState;
                    parsableString.index++;
                    numberOfParsedCharactersRead_SinceLastFreePassState++;
                }

                // If the next character in the string to test does not coincide with the next state - go to the last state representing '*'. If there are none - rejection!
                else if (runnerState.lastFreeRangeState != null) {
                    runnerState = runnerState.lastFreeRangeState;
                    parsableString.index++;
                    numberOfParsedCharactersRead_SinceLastFreePassState++;
                }
                else {
                    if (debugging) {
                        out.println("the next state does not represent the same character as the next character in the string to test, and there are no last-free-range-state - rejection!");
                    }
                    return false;
                }
            }
        }
        if (debugging) {
            out.println("finished reading parsing string and not at any final state - rejection!");
        }
        return false;
    }
939037
function make_lex( $i = 0 ){
		$Lex = new LexBuilder( $i );
		foreach( $this->collect_symbols() as $symbols ){
			foreach( $symbols as $t => $s ){
				if( preg_match('/^\W/', $s, $r ) ){
					$Lex->define_literal( (string) $t );
				}
				else if( $Lex->defined($s) ){
				}
				else if( defined($s) ){
					$Lex->redefine( $s );
				}
				else {
					$Lex->define( $t );
				}
			}
		}
		return $Lex;
	}
510811
function gettext($string)
    {
        global $PPHP;
        $req = grab('request');

        $default = $PPHP['config']['global']['default_lang'];
        $current = $req['lang'];

        $res = $string;
        try {
            if (self::$_gettext !== null) {
                $res = self::$_gettext->gettext($string);
            };
        } catch (\Exception $e) {
            echo $e->getMessage();
        };
        return $res;
    }
340610
def analyse(string = nil)
      raise(ArgumentError, 'Lexer: failed to start analysis: no source given!') unless
        string || @scanner

      self.data = string || @scanner.string

      until @scanner.eos?
        send("parse_#{MODE[@mode]}")
      end

      push([false, '$end'])
    end
1279670
function parseLoopStatement (input) {
  let current = 0
  let char = input[current]

  // parse through keys `each **foo, bar** in x`, which is everything before
  // the word "in"
  const keys = []
  let key = ''
  while (!`${char}${lookahead(3)}`.match(/\s(in|of)\s/)) {
    key += char
    next()

    // if we hit a comma, we're on to the next key
    if (char === ',') {
      keys.push(key.trim())
      key = ''
      next()
    }

    // if we reach the end of the string without getting "in/of", it's an error
    if (typeof char === 'undefined') {
      throw new Error("Loop statement lacking 'in' or 'of' keyword")
    }
  }
  keys.push(key.trim())

  // detect whether it's an in or of
  const loopType = lookahead(2)

  // space before, in/of, space after
  next(4)

  // the rest of the string is evaluated as the array/object to loop
  let expression = ''
  while (current < input.length) {
    expression += char
    next()
  }

  return {keys, expression, loopType}

  // Utility: Move to the next character in the parse
  function next (n = 1) {
    for (let i = 0; i < n; i++) { char = input[++current] }
  }

  // Utility: looks ahead n characters and returns the result
  function lookahead (n) {
    let counter = current
    const target = current + n
    let res = ''
    while (counter < target) {
      res += input[++counter]
    }
    return res
  }
}
1033156
function lex(phrase) {
        var words = phrase && typeof(phrase) === 'string' ? lexer.lex(phrase) : {status:"Invalid phrase parameter"};
        logger.log('verbose', '%s|lex|phrase=%s|words=%j', meta.module, phrase, words, meta);
        return words;
    }
637425
function Lexer(str, options) {
  options = options || {};
  this.stash = [];
  this.indentStack = [];
  this.indentRe = null;
  this.lineno = 1;
  this.column = 1;

  // HACK!
  function comment(str, val, offset, s) {
    var inComment = s.lastIndexOf('/*', offset) > s.lastIndexOf('*/', offset)
      , commentIdx = s.lastIndexOf('//', offset)
      , i = s.lastIndexOf('\n', offset)
      , double = 0
      , single = 0;

    if (~commentIdx && commentIdx > i) {
      while (i != offset) {
        if ("'" == s[i]) single ? single-- : single++;
        if ('"' == s[i]) double ? double-- : double++;

        if ('/' == s[i] && '/' == s[i + 1]) {
          inComment = !single && !double;
          break;
        }
        ++i;
      }
    }

    return inComment
      ? str
      : ((val === ',' && /^[,\t\n]+$/.test(str)) ? str.replace(/\n/, '\r') : val + '\r');
  };

  // Remove UTF-8 BOM.
  if ('\uFEFF' == str.charAt(0)) str = str.slice(1);

  this.str = str
    .replace(/\s+$/, '\n')
    .replace(/\r\n?/g, '\n')
    .replace(/\\ *\n/g, '\r')
    .replace(/([,(:](?!\/\/[^ ])) *(?:\/\/[^\n]*|\/\*.*?\*\/)?\n\s*/g, comment)
    .replace(/\s*\n[ \t]*([,)])/g, comment);
}