**********************************
Parsing
********************************** get the regex to find the next token
490916
function getParseRegexForToken(token, config) {
        var a;
        switch (token) {
        case 'DDDD':
            return parseTokenThreeDigits;
        case 'YYYY':
        case 'GGGG':
        case 'gggg':
            return parseTokenFourDigits;
        case 'YYYYY':
        case 'GGGGG':
        case 'ggggg':
            return parseTokenSixDigits;
        case 'S':
        case 'SS':
        case 'SSS':
        case 'DDD':
            return parseTokenOneToThreeDigits;
        case 'MMM':
        case 'MMMM':
        case 'dd':
        case 'ddd':
        case 'dddd':
            return parseTokenWord;
        case 'a':
        case 'A':
            return getLangDefinition(config._l)._meridiemParse;
        case 'X':
            return parseTokenTimestampMs;
        case 'Z':
        case 'ZZ':
            return parseTokenTimezone;
        case 'T':
            return parseTokenT;
        case 'SSSS':
            return parseTokenDigits;
        case 'MM':
        case 'DD':
        case 'YY':
        case 'GG':
        case 'gg':
        case 'HH':
        case 'hh':
        case 'mm':
        case 'ss':
        case 'M':
        case 'D':
        case 'd':
        case 'H':
        case 'h':
        case 'm':
        case 's':
        case 'w':
        case 'ww':
        case 'W':
        case 'WW':
        case 'e':
        case 'E':
            return parseTokenOneOrTwoDigits;
        default :
            a = new RegExp(regexpEscape(unescapeFormat(token.replace('\\', '')), "i"));
            return a;
        }
    }
76862
public function nextToken()
    {
        if ($this->_input === null) {
            return null;
        }

        do {
            if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
                // It covers both cases a) there are no matches (preg_match(...) === 0)
                // b) error occured (preg_match(...) === FALSE)
                return null;
            }

            $str = $match[0][0];
            $pos = $match[0][1];
            $endpos = $pos + strlen($str);

            $this->_position = $endpos;

            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
        } while ($token === null); // try again if token is skipped

        return $token;
    }
46383
Token parseAtom() throws ParseException {
		int ch = this.read();
		Token tok = null;
		switch (ch) {
		case T_LPAREN:
			return this.processParen();
		case T_LPAREN2:
			return this.processParen2(); // '(?:'
		case T_CONDITION:
			return this.processCondition(); // '(?('
		case T_MODIFIERS:
			return this.processModifiers(); // (?modifiers ... )
		case T_INDEPENDENT:
			return this.processIndependent();
		case T_DOT:
			this.next(); // Skips '.'
			tok = Token.token_dot;
			break;

		/**
		 * char-class ::= '[' ( '^'? range ','?)+ ']' range ::= '\d' | '\w' |
		 * '\s' | category-block | range-char | range-char '-' range-char
		 * range-char ::= '\[' | '\]' | '\\' | '\' [,-efnrtv] | bmp-code |
		 * character-2 bmp-char ::= '\' 'u' [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]
		 * [0-9a-fA-F]
		 */
		case T_LBRACKET:
			return this.parseCharacterClass(true);
		case T_SET_OPERATIONS:
			return this.parseSetOperations();

		case T_BACKSOLIDUS:
			switch (this.chardata) {
			case 'd':
			case 'D':
			case 'w':
			case 'W':
			case 's':
			case 'S':
				tok = this.getTokenForShorthand(this.chardata);
				this.next();
				return tok;

			case 'e':
			case 'f':
			case 'n':
			case 'r':
			case 't':
			case 'u':
			case 'v':
			case 'x': {
				int ch2 = this.decodeEscaped();
				if (ch2 < 0x10000) {
					tok = Token.createChar(ch2);
				} else {
					tok = Token.createString(REUtil.decomposeToSurrogates(ch2));
				}
			}
				break;

			case 'c':
				return this.processBacksolidus_c();
			case 'C':
				return this.processBacksolidus_C();
			case 'i':
				return this.processBacksolidus_i();
			case 'I':
				return this.processBacksolidus_I();
			case 'g':
				return this.processBacksolidus_g();
			case 'X':
				return this.processBacksolidus_X();
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
				return this.processBackreference();

			case 'P':
			case 'p':
				int pstart = this.offset;
				tok = processBacksolidus_pP(this.chardata);
				if (tok == null)
					throw this.ex("parser.atom.5", pstart);
				break;

			default:
				tok = Token.createChar(this.chardata);
			}
			this.next();
			break;

		case T_CHAR:
			if (this.chardata == ']' || this.chardata == '{'
					|| this.chardata == '}')
				throw this.ex("parser.atom.4", this.offset - 1);
			tok = Token.createChar(this.chardata);
			int high = this.chardata;
			this.next();
			if (REUtil.isHighSurrogate(high) && this.read() == T_CHAR
					&& REUtil.isLowSurrogate(this.chardata)) {
				char[] sur = new char[2];
				sur[0] = (char) high;
				sur[1] = (char) this.chardata;
				tok = Token.createParen(Token.createString(new String(sur)), 0);
				this.next();
			}
			break;

		default:
			throw this.ex("parser.atom.4", this.offset - 1);
		}
		return tok;
	}
468776
private LiteralToken nextRegularExpressionLiteralToken() {
    LiteralToken token = scanner.nextRegularExpressionLiteralToken();
    lastSourcePosition = token.location.end;
    return token;
  }
930570
function _parseLeafToken(token) {
  const [, name, remainder] = token.match(LEAF_REGEX);

  return {
    name: name.toLowerCase(),
    params: _parseParams(remainder),
  };
}
990486
public Stream<HString> findAllPatterns(@NonNull Pattern regex) {
      return Streams.asStream(new Iterator<HString>() {
         Matcher m = regex.matcher(HString.this);
         int start = -1;
         int end = -1;

         private boolean advance() {
            if (start == -1) {
               if (m.find()) {
                  start = m.start();
                  end = m.end();
               }
            }
            return start != -1;
         }

         @Override
         public boolean hasNext() {
            return advance();
         }

         @Override
         public HString next() {
            if (!advance()) {
               throw new NoSuchElementException();
            }
            HString sub = substring(start, end);
            start = -1;
            end = -1;
            //If we have tokens expand the match to the overlaping tokens.
            if (document() != null && document().isCompleted(Types.TOKEN)) {
               return union(sub.tokens());
            }
            return sub;
         }
      });
   }
97668
function _match($token)
	{
		switch($token) {
			case "+":
			case "-":
			case "*":
			case "/":
			case "(":
			case ")":
			case ",":
			case ";":
			case ">=":
			case "<=":
			case "=":
			case "<>":
			case "^":
			case "&":
			case "%":
				return $token;
				break;
			case ">":
				if ($this->_lookahead == '=') { // it's a GE token
					break;
				}
				return $token;
				break;
			case "<":
				// it's a LE or a NE token
				if (($this->_lookahead == '=') or ($this->_lookahead == '>')) {
					break;
				}
				return $token;
				break;
			default:
				// if it's a reference A1 or $A$1 or $A1 or A$1
				if (preg_match('/^\$?[A-Ia-i]?[A-Za-z]\$?[0-9]+$/',$token) and
				   !preg_match("/[0-9]/",$this->_lookahead) and
				   ($this->_lookahead != ':') and ($this->_lookahead != '.') and
				   ($this->_lookahead != '!'))
				{
					return $token;
				}
				// If it's an external reference (Sheet1!A1 or Sheet1:Sheet2!A1 or Sheet1!$A$1 or Sheet1:Sheet2!$A$1)
				elseif (preg_match("/^" . self::REGEX_SHEET_TITLE_UNQUOTED . "(\:" . self::REGEX_SHEET_TITLE_UNQUOTED . ")?\!\\$?[A-Ia-i]?[A-Za-z]\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead) and
					   ($this->_lookahead != ':') and ($this->_lookahead != '.'))
				{
					return $token;
				}
				// If it's an external reference ('Sheet1'!A1 or 'Sheet1:Sheet2'!A1 or 'Sheet1'!$A$1 or 'Sheet1:Sheet2'!$A$1)
				elseif (preg_match("/^'" . self::REGEX_SHEET_TITLE_QUOTED . "(\:" . self::REGEX_SHEET_TITLE_QUOTED . ")?'\!\\$?[A-Ia-i]?[A-Za-z]\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead) and
					   ($this->_lookahead != ':') and ($this->_lookahead != '.'))
				{
					return $token;
				}
				// if it's a range A1:A2 or $A$1:$A$2
				elseif (preg_match('/^(\$)?[A-Ia-i]?[A-Za-z](\$)?[0-9]+:(\$)?[A-Ia-i]?[A-Za-z](\$)?[0-9]+$/', $token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's an external range like Sheet1!A1:B2 or Sheet1:Sheet2!A1:B2 or Sheet1!$A$1:$B$2 or Sheet1:Sheet2!$A$1:$B$2
				elseif (preg_match("/^" . self::REGEX_SHEET_TITLE_UNQUOTED . "(\:" . self::REGEX_SHEET_TITLE_UNQUOTED . ")?\!\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+:\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's an external range like 'Sheet1'!A1:B2 or 'Sheet1:Sheet2'!A1:B2 or 'Sheet1'!$A$1:$B$2 or 'Sheet1:Sheet2'!$A$1:$B$2
				elseif (preg_match("/^'" . self::REGEX_SHEET_TITLE_QUOTED . "(\:" . self::REGEX_SHEET_TITLE_QUOTED . ")?'\!\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+:\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's a number (check that it's not a sheet name or range)
				elseif (is_numeric($token) and
						(!is_numeric($token.$this->_lookahead) or ($this->_lookahead == '')) and
						($this->_lookahead != '!') and ($this->_lookahead != ':'))
				{
					return $token;
				}
				// If it's a string (of maximum 255 characters)
				elseif (preg_match("/\"([^\"]|\"\"){0,255}\"/",$token) and $this->_lookahead != '"' and (substr_count($token, '"')%2 == 0))
				{
					return $token;
				}
			    // If it's an error code
			    elseif (preg_match("/^#[A-Z0\/]{3,5}[!?]{1}$/", $token) or $token == '#N/A')
			    {
			        return $token;
			    }
				// if it's a function call
				elseif (preg_match("/^[A-Z0-9\xc0-\xdc\.]+$/i",$token) and ($this->_lookahead == "("))
				{
					return $token;
				}
				//	It's an argument of some description (e.g. a named range),
				//		precise nature yet to be determined
				elseif(substr($token,-1) == ')') {
					return $token;
				}
				return '';
		}
	}
778472
private void findTokens() {
        mergedTokens = new TokenList();
        variable = null;
        state = TokenizerState.START;
        index = -1;

        while (index < rawTokens.length) {
            switch (state) {
            case START:
                next(TokenizerState.OUTSIDE_VARIABLE_CONTEXT);
                break;
            case OUTSIDE_VARIABLE_CONTEXT:
                handleTokenOutsideOfVariableContext();
                break;
            case INSIDE_VARIABLE_CONTEXT:
                handleTokenInsideOfVariableContext();
                break;
            default:
                break;
            }
        }
    }
73099
function zoneParser (span) {
  if (whitespace.test (span[0])) { return ["", span]; }
  var tokens = [];
  var sep = [];     // Separation indices.
  var section = toplevel (span);   // textual zones.
  while (section.zone !== null) {
    var zone = section.zone;
    // Add tokens before the zone.
    tokens = tokens.concat (span.slice (0, zone.from)
                                .split (whitespace)
                                .filter (nonEmpty));
    // Add the zone.
    tokens.push (span.slice (zone.from + 2, zone.to - 2));
    // Prepare for next iteration.
    span = span.slice (zone.to);
    section = toplevel (span);
  }
  tokens = tokens.concat (span.slice (0)
                              .split (whitespace)
                              .filter (nonEmpty));
  return tokens;
}
1255203
private static String nextToken(String[] tokens, int x, String context)
            throws ParseException {
        if (x < tokens.length) {
            return tokens[x];
        }
        throw new ParseException("missing token at end of '" + context + "'", -1);
    }
79538
public HString findPattern(@NonNull Pattern regex) {
      Matcher m = matcher(regex);
      if (m.find()) {
         return union(substring(m.start(), m.end()).tokens());
      }
      return Fragments.empty(document());
   }
1009685
function getAllMatchAndRange(regex, string) {
  if (!regex.global) {
    return [];
  }

  let resultArray = [];
  regex.lastIndex = 0;
  let result = regex.exec(string);
  while (result !== null) {
    resultArray.push([result[0], result.index, regex.lastIndex]);
    result = regex.exec(string);
  }
  return resultArray;
}
98237
function parseTag(position, tokens) {
	let token;
	let tag = {
		name : null,
		self : false,
		open : true
	};

	while (token = tokens[position++]) {
		// tag's end found so parsing must stop
		if (token.type === TokenTypes.TagEnd) {
			break;
		}

		// save the tag's name and continue parsing
		if (!tag.name && token.type === TokenTypes.TagName) {
			tag.name = token.value;
			continue;
		}

		// if a "/" is found after the tag's name, then it's a self-closing tag
		if (token.type === TokenTypes.TagClose) {
			tag.self = tag.open = !!tag.name;
		}
	}

	return tag;
}
8470
def find_tokens(self, string, **kwargs):
        
        # "The cat purs." => ["The cat purs ."]
        return find_tokens(string,
                punctuation = kwargs.get(  "punctuation", PUNCTUATION),
              abbreviations = kwargs.get("abbreviations", ABBREVIATIONS),
                    replace = kwargs.get(      "replace", replacements),
                  linebreak = r"\n{2,}")
874148
def next_token(self):
        """"""
        if self.lookahead:
            self.current_token = self.lookahead.popleft()
            return self.current_token

        self.current_token = self._parse_next_token()
        return self.current_token
20481
public function NextToken()
	{
		$anchor = $this->_offset;
		$ch = '\0';
		$this->Value = "";
		while ( $this->_offset < strlen( $this->_text ) )
		{
			$ch = $this->_text[ $this->_offset ];
			if ( preg_match( "/\s/", $ch ) && $ch != '\n' && $ch != '\r' )
				$this->_offset++;
			else
				break;
		}
		switch ( $ch )
		{
			case '\0':

				$this->Token = $ch;
				break;

			case '\n':

				$this->_offset++;
				if ($this->_offset < strlen( $this->_text ) - 1 && $this->_text[ $this->_offset ] == '\r' )
					$this->_offset++;
				$this->LineCount++;
				$this->Token = $ch;
				break;

			case '\r':

				$this->_offset++;
				if ( $this->_offset < strlen( $this->_text.Length ) - 1 && $this->_text[ $this->_offset ] == '\n')
					$this->_offset++;
				$this->LineCount++;
				$this->Token = '\n';
				break;

			default:

				if ( is_numeric( $ch ) )
				{
					$sb = array();
					while ( $this->_offset < strlen( $this->_text ) )
					{
						$ch = $this->_text[ $this->_offset ];
						if ( ! is_numeric( $ch ) )
							break;

						$sb[] = $ch;
						$this->_offset++;
					}
					$this->Token = StringTokenizer::TokenInt;
					$this->Value = implode( "", $sb );
				}
				else
				{
					$this->Token = $ch;
					$this->_offset++;
				}
				break;

		}
		return $this->Token;
	}
692596
protected function _getNextToken()
    {
        $this->token      = self::EOF;
        $this->tokenValue = null;
        $this->_eatWhitespace();

        if ($this->offset >= $this->sourceLength) {
            return(self::EOF);
        }

        $str       = $this->source;
        $strLength = $this->sourceLength;
        $i         = $this->offset;
        $start     = $i;

        switch ($str{$i}) {
            case '{':
                $this->token = self::LBRACE;
                break;
            case '}':
                $this->token = self::RBRACE;
                break;
            case '[':
                $this->token = self::LBRACKET;
                break;
            case ']':
                $this->token = self::RBRACKET;
                break;
            case ',':
                $this->token = self::COMMA;
                break;
            case ':':
                $this->token = self::COLON;
                break;
            case '"':
                $result = '';
                do {
                    $i++;
                    if ($i >= $strLength) {
                        break;
                    }

                    $chr = $str{$i};

                    if ($chr == '\\') {
                        $i++;
                        if ($i >= $strLength) {
                            break;
                        }
                        $chr = $str{$i};
                        switch ($chr) {
                            case '"':
                                $result .= '"';
                                break;
                            case '\\':
                                $result .= '\\';
                                break;
                            case '/':
                                $result .= '/';
                                break;
                            case 'b':
                                $result .= "\x08";
                                break;
                            case 'f':
                                $result .= "\x0c";
                                break;
                            case 'n':
                                $result .= "\x0a";
                                break;
                            case 'r':
                                $result .= "\x0d";
                                break;
                            case 't':
                                $result .= "\x09";
                                break;
                            case '\'':
                                $result .= '\'';
                                break;
                            default:
                                throw new RuntimeException("Illegal escape sequence '{$chr}'");
                        }
                    } elseif ($chr == '"') {
                        break;
                    } else {
                        $result .= $chr;
                    }
                } while ($i < $strLength);

                $this->token = self::DATUM;
                //$this->tokenValue = substr($str, $start + 1, $i - $start - 1);
                $this->tokenValue = $result;
                break;
            case 't':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "true") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = true;
                $i += 3;
                break;
            case 'f':
                if (($i+ 4) < $strLength && substr($str, $start, 5) == "false") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = false;
                $i += 4;
                break;
            case 'n':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "null") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = null;
                $i += 3;
                break;
        }

        if ($this->token != self::EOF) {
            $this->offset = $i + 1; // Consume the last token character
            return($this->token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match('/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s', $str, $matches, PREG_OFFSET_CAPTURE, $start) && $matches[0][1] == $start) {
                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new RuntimeException("Octal notation not supported by JSON (value: {$datum})");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new RuntimeException("Illegal number format: {$datum}");
                }

                $this->token = self::DATUM;
                $this->offset = $start + strlen($datum);
            }
        } else {
            throw new RuntimeException('Illegal Token');
        }

        return $this->token;
    }