Returns the matched text, and moves to the next token
186023
def _scan(self, type):
        
        tok = self._scanner.token(self._pos, frozenset([type]))
        if tok[2] != type:
            err = SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type))
            err.pos = tok[0]
            raise err
        self._pos += 1
        return tok[3]
1050147
public String getPosFromMatchResult(int tokBegin, int tokEnd, Sentence s, JCas jcas) {
		// get all tokens in sentence
		HashMap<Integer, Token> hmTokens = new HashMap<Integer, Token>();
		FSIterator iterTok = jcas.getAnnotationIndex(Token.type).subiterator(s);
		while (iterTok.hasNext()) {
			Token token = (Token) iterTok.next();
			hmTokens.put(token.getBegin(), token);
		}
		// get correct token
		String pos = "";
		if (hmTokens.containsKey(tokBegin)) {
			Token tokenToCheck = hmTokens.get(tokBegin);
			pos = tokenToCheck.getPos();
		}
		return pos;
	}
97668
function _match($token)
	{
		switch($token) {
			case "+":
			case "-":
			case "*":
			case "/":
			case "(":
			case ")":
			case ",":
			case ";":
			case ">=":
			case "<=":
			case "=":
			case "<>":
			case "^":
			case "&":
			case "%":
				return $token;
				break;
			case ">":
				if ($this->_lookahead == '=') { // it's a GE token
					break;
				}
				return $token;
				break;
			case "<":
				// it's a LE or a NE token
				if (($this->_lookahead == '=') or ($this->_lookahead == '>')) {
					break;
				}
				return $token;
				break;
			default:
				// if it's a reference A1 or $A$1 or $A1 or A$1
				if (preg_match('/^\$?[A-Ia-i]?[A-Za-z]\$?[0-9]+$/',$token) and
				   !preg_match("/[0-9]/",$this->_lookahead) and
				   ($this->_lookahead != ':') and ($this->_lookahead != '.') and
				   ($this->_lookahead != '!'))
				{
					return $token;
				}
				// If it's an external reference (Sheet1!A1 or Sheet1:Sheet2!A1 or Sheet1!$A$1 or Sheet1:Sheet2!$A$1)
				elseif (preg_match("/^" . self::REGEX_SHEET_TITLE_UNQUOTED . "(\:" . self::REGEX_SHEET_TITLE_UNQUOTED . ")?\!\\$?[A-Ia-i]?[A-Za-z]\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead) and
					   ($this->_lookahead != ':') and ($this->_lookahead != '.'))
				{
					return $token;
				}
				// If it's an external reference ('Sheet1'!A1 or 'Sheet1:Sheet2'!A1 or 'Sheet1'!$A$1 or 'Sheet1:Sheet2'!$A$1)
				elseif (preg_match("/^'" . self::REGEX_SHEET_TITLE_QUOTED . "(\:" . self::REGEX_SHEET_TITLE_QUOTED . ")?'\!\\$?[A-Ia-i]?[A-Za-z]\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead) and
					   ($this->_lookahead != ':') and ($this->_lookahead != '.'))
				{
					return $token;
				}
				// if it's a range A1:A2 or $A$1:$A$2
				elseif (preg_match('/^(\$)?[A-Ia-i]?[A-Za-z](\$)?[0-9]+:(\$)?[A-Ia-i]?[A-Za-z](\$)?[0-9]+$/', $token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's an external range like Sheet1!A1:B2 or Sheet1:Sheet2!A1:B2 or Sheet1!$A$1:$B$2 or Sheet1:Sheet2!$A$1:$B$2
				elseif (preg_match("/^" . self::REGEX_SHEET_TITLE_UNQUOTED . "(\:" . self::REGEX_SHEET_TITLE_UNQUOTED . ")?\!\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+:\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's an external range like 'Sheet1'!A1:B2 or 'Sheet1:Sheet2'!A1:B2 or 'Sheet1'!$A$1:$B$2 or 'Sheet1:Sheet2'!$A$1:$B$2
				elseif (preg_match("/^'" . self::REGEX_SHEET_TITLE_QUOTED . "(\:" . self::REGEX_SHEET_TITLE_QUOTED . ")?'\!\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+:\\$?([A-Ia-i]?[A-Za-z])?\\$?[0-9]+$/u",$token) and
					   !preg_match("/[0-9]/",$this->_lookahead))
				{
					return $token;
				}
				// If it's a number (check that it's not a sheet name or range)
				elseif (is_numeric($token) and
						(!is_numeric($token.$this->_lookahead) or ($this->_lookahead == '')) and
						($this->_lookahead != '!') and ($this->_lookahead != ':'))
				{
					return $token;
				}
				// If it's a string (of maximum 255 characters)
				elseif (preg_match("/\"([^\"]|\"\"){0,255}\"/",$token) and $this->_lookahead != '"' and (substr_count($token, '"')%2 == 0))
				{
					return $token;
				}
			    // If it's an error code
			    elseif (preg_match("/^#[A-Z0\/]{3,5}[!?]{1}$/", $token) or $token == '#N/A')
			    {
			        return $token;
			    }
				// if it's a function call
				elseif (preg_match("/^[A-Z0-9\xc0-\xdc\.]+$/i",$token) and ($this->_lookahead == "("))
				{
					return $token;
				}
				//	It's an argument of some description (e.g. a named range),
				//		precise nature yet to be determined
				elseif(substr($token,-1) == ')') {
					return $token;
				}
				return '';
		}
	}
1245764
def parse(self, text):
        """"""
        text = self.preprocess(text)
        token_stack = []
        last_pos = 0

        # Iterate through all matched tokens
        for match in self.regex.finditer(text):
            # Find which token has been matched by regex
            token, match_type, group = self.get_matched_token(match)

            # Get params from stack of tokens
            params = self.get_params(token_stack)

            # Should we skip interpreting tokens?
            skip = token_stack[-1].skip if token_stack else False

            # Check for end token first
            if match_type == MatchType.end:
                if not skip or token_stack[-1] == token:
                    removed = self.remove_token(token_stack, token)
                    if removed:
                        skip = False
                    else:
                        skip = True

            if not skip:
                # Append text preceding matched token
                start_pos = match.start(group)
                if start_pos > last_pos:
                    yield Segment(self.postprocess(text[last_pos:start_pos]), **params)

                # Actions specific for start token or single token
                if match_type == MatchType.start:
                    token_stack.append(token)
                elif match_type == MatchType.single:
                    single_params = params.copy()
                    single_params.update(token.params)
                    single_text = token.text if token.text is not None else match.group(group)
                    yield Segment(single_text, token=token, match=match, **single_params)

                # Move last position pointer to the end of matched token
                last_pos = match.end(group)

        # Append anything that's left
        if last_pos < len(text):
            params = self.get_params(token_stack)
            yield Segment(self.postprocess(text[last_pos:]), **params)
604151
def lex(self, text):
        """"""
        for match in self.regex.finditer(text):
            name = match.lastgroup
            yield (name, match.group(name))
382198
def next_match char
      data = get_content
      row = focussed_index + 1
      row.upto(data.length-1) do |ix|
        #val = data[ix].chomp rescue return  # 2010-01-05 15:28 crashed on trueclass
        val = data[ix].to_s rescue return  # 2010-01-05 15:28 crashed on trueclass
        #if val[0,1] == char #and val != currval
        if val[0,1].casecmp(char) == 0 #AND VAL != CURRval
          return ix
        end
      end
      row = focussed_index - 1
      0.upto(row) do |ix|
        #val = data[ix].chomp
        val = data[ix].to_s
        #if val[0,1] == char #and val != currval
        if val[0,1].casecmp(char) == 0 #and val != currval
          return ix
        end
      end
      return -1
    end
716828
def _MatchDirective(token):
    """"""
    # Tokens below must start with '.'
    if token.startswith('.'):
        token = token[1:]
    else:
        return None, None
    
    if token == 'end':
        return END_TOKEN, None
    
    if token == 'alternates with':
        return ALTERNATES_TOKEN, token
    
    if token.startswith('or'):
        if token.strip() == 'or':
            return OR_TOKEN, None
        else:
            pred_str = token[2:].strip()
            return OR_TOKEN, pred_str
    
    match = _SECTION_RE.match(token)
    if match:
        repeated, section_name = match.groups()
        if repeated:
            return REPEATED_SECTION_TOKEN, section_name
        else:
            return SECTION_TOKEN, section_name
    
    if token.startswith('template '):
        return SUBST_TEMPLATE_TOKEN, token[9:].strip()
    if token.startswith('define '):
        return DEF_TOKEN, token[7:].strip()
    
    if token.startswith('if '):
        return IF_TOKEN, token[3:].strip()
    if token.endswith('?'):
        return PREDICATE_TOKEN, token
    
    return None, None
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
1053497
def _parse_next_token(self):
        """"""
        while self._position < self.limit:
            token = self._next_pattern()
            if token:
                return token

        return None
330183
def apply(self, tokens):
        
        # Note: we could also scan for patterns, e.g.,
        # "my|his|her name is|was *" => NNP-PERS.
        i = 0
        while i < len(tokens):
            w = tokens[i][0].lower()
            if RE_ENTITY1.match(w) \
            or RE_ENTITY2.match(w) \
            or RE_ENTITY3.match(w):
                tokens[i][1] = self.tag
            if w in self:
                for e in self[w]:
                    # Look ahead to see if successive words match the named entity.
                    e, tag = (e[:-1], "-"+e[-1].upper()) if e[-1] in self._cmd else (e, "")
                    b = True
                    for j, e in enumerate(e):
                        if i + j >= len(tokens) or tokens[i+j][0].lower() != e:
                            b = False; break
                    if b:
                        for token in tokens[i:i+j+1]:
                            token[1] = token[1] if token[1].startswith(self.tag) else self.tag
                            token[1] += tag
                        i += j
                        break
            i += 1
        return tokens
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
1167740
static function matchText( &$text, $match_pre, $match_post )
    {
        $ret = null;
        if ( $match_pre !== '' )
        {
            $pos = strpos( $text, $match_pre );
            if ( $pos === false )
                return null;

            $ret = substr( $text, $pos + strlen( $match_pre ) );
            $text = substr( $text, 0, $pos );
        }
        if ( $match_post !== '' )
        {
            $pos = strpos( $ret, $match_post );
            if ( $pos === false )
                return null;

            $text .= substr( $ret, $pos + 1 );
            $ret = substr( $ret, 0, $pos );
        }
        return $ret;
    }
454068
private Token lookahead(int n) {
        Token current = token;
        for (int i = 0; i < n; i++) {
            if (current.next == null) {
                current.next = token_source.getNextToken();
            }
            current = current.next;
        }
        return current;
    }
1123667
def next(self):
        
        line = self.buffer.readline()

        while line == '\n':       # Skip forward to the next line with content.
            line = self.buffer.readline()

        if line == '':            # End of file.
            self.at_eof = True
            return None, None

        match = re.match('([A-Z]{2}|[C][1])\W(.*)', line)
        if match is not None:
            self.current_tag, data = match.groups()
        else:
            self.current_tag = self.last_tag
            data = line.strip()
        return self.current_tag, _cast(data)
358926
function matchKeyword(keyword) {
        var token = lookahead();
        return token.type === Token.Keyword && token.value === keyword;
    }
1363595
@Override
    public int matches(Replaceable text,
            int[] offset,
            int limit,
            boolean incremental) {

        if (offset[0] == limit) {
            // Strings, if any, have length != 0, so we don't worry
            // about them here.  If we ever allow zero-length strings
            // we much check for them here.
            if (contains(UnicodeMatcher.ETHER)) {
                return incremental ? U_PARTIAL_MATCH : U_MATCH;
            } else {
                return U_MISMATCH;
            }
        } else {
            if (strings.size() != 0) { // try strings first

                // might separate forward and backward loops later
                // for now they are combined

                // TODO Improve efficiency of this, at least in the forward
                // direction, if not in both.  In the forward direction we
                // can assume the strings are sorted.

                boolean forward = offset[0] < limit;

                // firstChar is the leftmost char to match in the
                // forward direction or the rightmost char to match in
                // the reverse direction.
                char firstChar = text.charAt(offset[0]);

                // If there are multiple strings that can match we
                // return the longest match.
                int highWaterLength = 0;

                for (String trial : strings) {
                    //if (trial.length() == 0) {
                    //    return U_MATCH; // null-string always matches
                    //}
                    // assert(trial.length() != 0); // We ensure this elsewhere

                    char c = trial.charAt(forward ? 0 : trial.length() - 1);

                    // Strings are sorted, so we can optimize in the
                    // forward direction.
                    if (forward && c > firstChar) break;
                    if (c != firstChar) continue;

                    int length = matchRest(text, offset[0], limit, trial);

                    if (incremental) {
                        int maxLen = forward ? limit-offset[0] : offset[0]-limit;
                        if (length == maxLen) {
                            // We have successfully matched but only up to limit.
                            return U_PARTIAL_MATCH;
                        }
                    }

                    if (length == trial.length()) {
                        // We have successfully matched the whole string.
                        if (length > highWaterLength) {
                            highWaterLength = length;
                        }
                        // In the forward direction we know strings
                        // are sorted so we can bail early.
                        if (forward && length < highWaterLength) {
                            break;
                        }
                        continue;
                    }
                }

                // We've checked all strings without a partial match.
                // If we have full matches, return the longest one.
                if (highWaterLength != 0) {
                    offset[0] += forward ? highWaterLength : -highWaterLength;
                    return U_MATCH;
                }
            }
            return super.matches(text, offset, limit, incremental);
        }
    }
5347
function (source, nextToken) {
      var tm = this,
          tokenizerRe,
          lastIndex,
          matches,
          textType;

      // Initialize for first run
      // if no template passed assume first pass
      // and use the main template.
      if (typeof source === "undefined") {
        source = tm.template;
      }

      // if no token passed, this is an open tag lookup
      if (typeof nextToken === "undefined") {
        tm._tokens = [];
        nextToken = tm.lex.tagOpen;
      }

      while (source) {
        // Use created token.
        tokenizerRe = nextToken.compiledRe;
        matches = tokenizerRe.exec(source);

        lastIndex = tokenizerRe.lastIndex;
        tokenizerRe.lastIndex = 0;

        // This guy matched. So let's push the tokens.
        if (matches) {

          // Flags to see if code or text
          if (nextToken === tm.lex.tagOpen) {
            // if the match was done using the tagOpen, then everything we matched is normal text.
            textType = "text";
          } else {
            // assume the match was a closing tag, so everything we have is code.
            textType = "code";
          }

          // if the token is a tagOpen, or tagClose, we need to push it with the text value matched.
          if (nextToken.hasText) {
            tm._tokens.push({
              type : textType,
              value : matches[1]
            });
          } else {
            // push the token name into the array to mark the open/close/print status on it.
            tm._tokens.push({
              type : nextToken.name
            });
          }

          // A print indicator just shows open prints, so we buffer the open print.
          if (nextToken === tm.lex.printIndicator) {
            tm._openPrints.push(tm._openBrackets);
          }

          // Closing an open print needs to count for open brackets, so we can separate the code blocks.
          if (nextToken === tm.lex.tagClose && tm._openPrints.length > 0) {
            tm._countBrackets(matches[1]);
          }

          // We matched, so let's keep going.
          // from where we left with our new next token.
          source    = source.substr(lastIndex);
          nextToken = tm._nextToken(nextToken);

        //No match:
        } else {
          // We couldn't match anything else. So: Either it's a syntax error,
          // or it's just text. If we were awaiting another token we fail, otherwise
          // assume it's just text.
          if (nextToken == tm.lex.tagOpen) {
            tm._tokens.push({
              type  : "text",
              value : source
            });

            source = "";

          } else {
            // If it wasn't optional. GO!
            if (nextToken.optional) {
              // So continue from where we left with our new next token.
              source    = source.substr(lastIndex);
              nextToken = tm._nextToken(nextToken);

            // so it was not a printing tag, so something went wrong.
            } else {
              throw "Syntax Error: Expecting " + nextToken.name + " ... Nowhere to be found. So I was like, what?";
            }
          }
        }
      }
    }