Tokenization rules
489251
public function setupRules()
    {
        $identifier = '-?[_a-zA-Z]+[_a-zA-Z0-9-]*';
        $at = [
            'charset', 'import', 'namespace',
            'media', 'supports', 'document', 'page', 'font-face', 'keyframes', 'viewport', 'counter-style',
            'font-feature-values', 'swash', 'ornaments', 'annotation', 'stylistic', 'styleset', 'character-variant'
        ];
        
        $this->rules->addMany([
            'string' => CommonFeatures::strings(['single' => '\'', 'double' => '"'], [
                'context' => $this->everywhere()
            ]),

            'meta.declaration' => [
                new OpenRule(new SubStringMatcher('{'), [
                    'context' => ['!meta.declaration.media', '!comment'],
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
                new CloseRule(new SubStringMatcher('}')),
            ],

            'meta.declaration.media' => [
                new Rule(new RegexMatcher('/@media(.*?\{)/'), [
                    'context' => Validator::everywhere(),
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
            ],

            'meta.declaration.rule' => [
                new OpenRule(new RegexMatcher('/@media.*(\()/'), [
                    'context' => ['meta.declaration.media'],
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
                new CloseRule(new SubStringMatcher(')')),
            ],

            'keyword.at-rule' => new Rule(new RegexMatcher('/(@(?:-[a-z]+-)?(?:'.implode('|', $at).'))/'), [
                'priority' => 2
            ]),

            'symbol.selector.id'    => new Rule(new RegexMatcher("/(#$identifier)/i")),
            'symbol.selector.tag'   => new Rule(new RegexMatcher('/(?>[\s}]|^)(?=(\w+)[^;]*\{)/ms')),
            'symbol.selector.class' => new Rule(new RegexMatcher("/(\\.$identifier)/i")),

            'symbol.selector.class.pseudo' => new Rule(new RegexMatcher("/(:{1,2}$identifier)/")),

            'number' => new Rule(new RegexMatcher("/([-+]?[0-9]*\\.?[0-9]+([\\w%]+)?)/"), [
                'context'  => ['meta.declaration', '!constant.color', '!comment', '!symbol', '!comment', '!string'],
                'priority' => 0
            ]),
            
            'symbol.property' => new Rule(new RegexMatcher("/($identifier:)/"), [
                'context' => ['meta.declaration', '!symbol', '!comment'],
                'priority' => 2
            ]),

            'call' => new Rule(new RegexMatcher("/($identifier)\\s*\\((?:(?P<string>[a-z].*?)|.*?)\\)/", [
                1 => Token::NAME,
                'string' => 'string.argument'
            ]), [
                'context' => ['meta.declaration', '!comment', '!string', '!keyword']
            ]),

            'constant.color' => [
                new Rule(new RegexMatcher("/(#[0-9a-f]{3,6})/i"), [
                    'priority' => 2,
                    'context'  => ['meta.declaration', '!comment']
                ]),
                new Rule(new WordMatcher([
                    'white', 'silver', 'gray', 'black', 'red', 'maroon', 'yellow', 'olive',
                    'lime', 'green', 'aqua', 'teal', 'blue', 'navy', 'fuchsia', 'purple'
                ]), [
                    'context'  => ['meta.declaration', '!comment', '!symbol', '!variable']
                ]),
            ],

            'operator' => new Rule(new WordMatcher(['>', '+', '*', '!important'], ['separated' => false]), [
                'context' => $this->everywhere()
            ]),

            'operator.punctuation' => new Rule(new SubStringMatcher(';'), [
                'context' => $this->everywhere()
            ]),

            'comment' => new Rule(new CommentMatcher([], [['/*', '*/']]), ['context' => $this->everywhere()])
        ]);
    }
586992
def tokeniter(self, source, name, filename=None, state=None):
        
        source = '\n'.join(unicode(source).splitlines())
        pos = 0
        lineno = 1
        stack = ['root']
        if state is not None and state != 'root':
            assert state in ('variable', 'block'), 'invalid state'
            stack.append(state + '_begin')
        else:
            state = 'root'
        statetokens = self.rules[stack[-1]]
        source_length = len(source)

        balancing_stack = []

        while 1:
            # tokenizer loop
            for regex, tokens, new_state in statetokens:
                m = regex.match(source, pos)
                # if no match we try again with the next rule
                if m is None:
                    continue

                # we only match blocks and variables if brances / parentheses
                # are balanced. continue parsing with the lower rule which
                # is the operator rule. do this only if the end tags look
                # like operators
                if balancing_stack and \
                   tokens in ('variable_end', 'block_end',
                              'linestatement_end'):
                    continue

                # tuples support more options
                if isinstance(tokens, tuple):
                    for idx, token in enumerate(tokens):
                        # failure group
                        if token.__class__ is Failure:
                            raise token(lineno, filename)
                        # bygroup is a bit more complex, in that case we
                        # yield for the current token the first named
                        # group that matched
                        elif token == '#bygroup':
                            for key, value in m.groupdict().iteritems():
                                if value is not None:
                                    yield lineno, key, value
                                    lineno += value.count('\n')
                                    break
                            else:
                                raise RuntimeError('%r wanted to resolve '
                                                   'the token dynamically'
                                                   ' but no group matched'
                                                   % regex)
                        # normal group
                        else:
                            data = m.group(idx + 1)
                            if data or token not in ignore_if_empty:
                                yield lineno, token, data
                            lineno += data.count('\n')

                # strings as token just are yielded as it.
                else:
                    data = m.group()
                    # update brace/parentheses balance
                    if tokens == 'operator':
                        if data == '{':
                            balancing_stack.append('}')
                        elif data == '(':
                            balancing_stack.append(')')
                        elif data == '[':
                            balancing_stack.append(']')
                        elif data in ('}', ')', ']'):
                            if not balancing_stack:
                                raise TemplateSyntaxError('unexpected \'%s\'' %
                                                          data, lineno, name,
                                                          filename)
                            expected_op = balancing_stack.pop()
                            if expected_op != data:
                                raise TemplateSyntaxError('unexpected \'%s\', '
                                                          'expected \'%s\'' %
                                                          (data, expected_op),
                                                          lineno, name,
                                                          filename)
                    # yield items
                    if data or tokens not in ignore_if_empty:
                        yield lineno, tokens, data
                    lineno += data.count('\n')

                # fetch new position into new variable so that we can check
                # if there is a internal parsing error which would result
                # in an infinite loop
                pos2 = m.end()

                # handle state changes
                if new_state is not None:
                    # remove the uppermost state
                    if new_state == '#pop':
                        stack.pop()
                    # resolve the new state by group checking
                    elif new_state == '#bygroup':
                        for key, value in m.groupdict().iteritems():
                            if value is not None:
                                stack.append(key)
                                break
                        else:
                            raise RuntimeError('%r wanted to resolve the '
                                               'new state dynamically but'
                                               ' no group matched' %
                                               regex)
                    # direct state name given
                    else:
                        stack.append(new_state)
                    statetokens = self.rules[stack[-1]]
                # we are still at the same position and no stack change.
                # this means a loop without break condition, avoid that and
                # raise error
                elif pos2 == pos:
                    raise RuntimeError('%r yielded empty string without '
                                       'stack change' % regex)
                # publish new function and start again
                pos = pos2
                break
            # if loop terminated without break we havn't found a single match
            # either we are at the end of the file or we have a problem
            else:
                # end of text
                if pos >= source_length:
                    return
                # something went wrong
                raise TemplateSyntaxError('unexpected char %r at %d' %
                                          (source[pos], pos), lineno,
                                          name, filename)
309445
def _parse_tokenize(self, rule):
        """"""

        for token in self._TOKENIZE_RE.split(rule):
            # Skip empty tokens
            if not token or token.isspace():
                continue

            # Handle leading parens on the token
            clean = token.lstrip('(')
            for i in range(len(token) - len(clean)):
                yield '(', '('

            # If it was only parentheses, continue
            if not clean:
                continue
            else:
                token = clean

            # Handle trailing parens on the token
            clean = token.rstrip(')')
            trail = len(token) - len(clean)

            # Yield the cleaned token
            lowered = clean.lower()
            if lowered in ('and', 'or', 'not'):
                # Special tokens
                yield lowered, clean
            elif clean:
                # Not a special token, but not composed solely of ')'
                if len(token) >= 2 and ((token[0], token[-1]) in
                                        [('"', '"'), ("'", "'")]):
                    # It's a quoted string
                    yield 'string', token[1:-1]
                else:
                    yield 'check', self._parse_check(clean)

            # Yield the trailing parens
            for i in range(trail):
                yield ')', ')'
1318278
def extract_token_and_qualifier(text, line=0, column=0):
    '''
    
    '''
    # Note: not using the tokenize module because text should be unicode and
    # line/column refer to the unicode text (otherwise we'd have to know
    # those ranges after converted to bytes).
    if line < 0:
        line = 0
    if column < 0:
        column = 0

    if isinstance(text, bytes):
        text = text.decode('utf-8')

    lines = text.splitlines()
    try:
        text = lines[line]
    except IndexError:
        return TokenAndQualifier(u'', u'')

    if column >= len(text):
        column = len(text)

    text = text[:column]
    token = u''
    qualifier = u''

    temp_token = []
    for i in range(column - 1, -1, -1):
        c = text[i]
        if c in identifier_part or isidentifier(c) or c == u'.':
            temp_token.append(c)
        else:
            break
    temp_token = u''.join(reversed(temp_token))
    if u'.' in temp_token:
        temp_token = temp_token.split(u'.')
        token = u'.'.join(temp_token[:-1])
        qualifier = temp_token[-1]
    else:
        qualifier = temp_token

    return TokenAndQualifier(token, qualifier)
764704
def as_tokens(ctx: List[ParserRuleContext]) -> List[str]:
    
    return [as_token(e) for e in ctx]
532175
public function setupRules()
    {
        $standard = new Validator(['!string', '!comment']);

        $this->rules->addMany([
            'keyword' => new Rule(new WordMatcher([
                'and', 'del', 'from', 'not', 'while', 'as', 'elif', 'global', 'or', 'with', 'assert', 'else', 'if',
                'pass', 'yield', 'break', 'except', 'import', 'print', 'class', 'exec', 'in', 'raise', 'continue',
                'finally', 'is', 'return', 'def', 'for', 'lambda', 'try',
            ])),

            'operator' => new Rule(
                new RegexMatcher('/([-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~])|\b(or|and|not)\b/'), [
                    'priority' => -1
                ]
            ),

            'expression' => new Rule(new RegexMatcher('/\{(\S+)\}/'), [
                'context' => ['string']
            ]),

            'variable' => [
                new Rule(new RegexMatcher('/[^\w.]([a-z_]\w*)\.\w/i')),
                'property' => new Rule(new RegexMatcher('/(?=(?:\w|\)|\])\s*\.([a-z_]\w*))/i'), [
                    'priority' => -2,
                    'context' => ['*none', '*expression']
                ]),
            ],

            'symbol' => [
                new Rule(new RegexMatcher('/import\s+([a-z_][\w.]*)(?:\s*,\s*([a-z_][\w.]*))*/i', [
                    1 => Token::NAME,
                    2 => Token::NAME,
                ])),
                'library' => new Rule(new RegexMatcher('/from\s+([a-z_][\w.]*)\s+import/i', [
                    1 => Token::NAME,
                ]))
            ],

            'operator.escape' => new Rule(new RegexMatcher('/(\\\(?:\R|.|[0-7]{3}|x\x{2}))/s'), [
                'context' => Validator::everywhere()
            ]),

            'comment' => new Rule(new CommentMatcher(['#'])),
            'constant.special' => [
                new Rule(new WordMatcher(['True', 'False', 'NotImplemented', 'Ellipsis'], [
                    'case-sensitivity' => true
                ])),
                new Rule(new RegexMatcher('/\b(__\w+__)\b/'))
            ],
            'call' => new Rule(new RegexMatcher('/([a-z_]\w*)\s*\(/i'), ['priority' => 2]),

            'meta.newline' => new CloseRule(new RegexMatcher('/()\R/'), [
                'factory' => new TokenFactory(TerminatorToken::class),
                'context' => ['!operator.escape', 'string.single-line'],
                'closes'  => ['string.single-line.double', 'string.single-line.single']
            ]),

            'number' => new Rule(
                new RegexMatcher('/(-?\b(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?)\b/')
            ),

            'string' => [
                'single-line' => [
                    'double' => new Rule(new RegexMatcher('/(?:^|[^"])(")(?=[^"]|$)/'), [
                        'factory' => new TokenFactory(ContextualToken::class),
                        'context' => $standard,
                    ]),
                    'single' => new Rule(new RegexMatcher('/(?:^|[^\'])(\')(?=[^\']|$)/'), [
                        'factory' => new TokenFactory(ContextualToken::class),
                        'context' => $standard,
                    ]),
                ],
                'multi-line' => [
                    'double' => new Rule(new SubStringMatcher('"""'), [
                        'factory'  => new TokenFactory(ContextualToken::class),
                        'context'  => $standard,
                        'priority' => 2,
                    ]),
                    'single' => new Rule(new SubStringMatcher('\'\'\''), [
                        'factory'  => new TokenFactory(ContextualToken::class),
                        'context'  => $standard,
                        'priority' => 2,
                    ]),
                ]
            ],

            'symbol.function' => new Rule(new RegexMatcher('/def\s+([a-z_]\w+)\s*\(/i')),
            'symbol.class'    => new Rule(new RegexMatcher('/class\s+([a-z_]\w+)/i')),
        ]);
    }
143301
def tokenize(self, data):
        '''
        

        '''
        super().tokenize(data)
        token_tuple_zip = self.n_gram.generate_tuple_zip(self.token, self.n)
        token_list = []
        self.token = ["".join(list(token_tuple)) for token_tuple in token_tuple_zip]
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
725291
void tokenize(String pat) throws javax.xml.transform.TransformerException
  {
    tokenize(pat, null);
  }
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
1090775
function Rules() {
		var rules = new Array(stateSize);
		for ( var j = 0; j < stateSize; j++) {
			rules[j] = new Array(stateSize);
		}
		rules[BEGIN.num][OPEN.num] = 0;
		rules[BEGIN.num][C_OPEN.num] = 1;
		rules[BEGIN.num][CHAR.num] = 2;
		rules[CHAR_END.num][CHAR.num] = 3;
		rules[BEGIN.num][CLOSE.num] = 4;
		rules[BEGIN.num][C_CLOSE.num] = 5;
		rules[BEGIN.num][CHAR_END.num] = 6;

		return function(topStack, lexToken) {
			return rules[topStack.num][lexToken.num];
		};
	}
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
743302
private function tokenize()
    {
        $this->tokenizeTitle();
        $this->tokenizeFields();
        $this->tokenizeBricks();
        $this->tokenizeDynamicBricks();
        $this->tokenVector = array_filter($this->tokenVector);
        arsort($this->tokenVector);
    }
1008273
private function _tokenize_rule($content, &$position)
    {
        $cond = strtolower(self::tokenize($content, 1, $position));
        if ($cond != 'if' && $cond != 'elsif' && $cond != 'else') {
            return null;
        }

        $disabled = false;
        $join     = false;
        $join_not = false;
        $length   = strlen($content);

        // disabled rule (false + comment): if false # .....
        if (preg_match('/^\s*false\s+#\s*/i', substr($content, $position, 20), $m)) {
            $position += strlen($m[0]);
            $disabled = true;
        }

        while ($position < $length) {
            $tokens    = self::tokenize($content, true, $position);
            $separator = array_pop($tokens);

            if (!empty($tokens)) {
                $token = array_shift($tokens);
            }
            else {
                $token = $separator;
            }

            $token = strtolower($token);

            if ($token == 'not') {
                $not = true;
                $token = strtolower(array_shift($tokens));
            }
            else {
                $not = false;
            }

            // we support "not allof" as a negation of allof sub-tests
            if ($join_not) {
                $not = !$not;
            }

            switch ($token) {
            case 'allof':
                $join     = true;
                $join_not = $not;
                break;

            case 'anyof':
                break;

            case 'size':
                $test = array('test' => 'size', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])
                        && preg_match('/^:(under|over)$/i', $tokens[$i])
                    ) {
                        $test['type'] = strtolower(substr($tokens[$i], 1));
                    }
                }

                $tests[] = $test;
                break;

            case 'header':
            case 'string':
            case 'address':
            case 'envelope':
                $test = array('test' => $token, 'not' => $not);

                $test['arg2'] = array_pop($tokens);
                $test['arg1'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                if ($token != 'header' && $token != 'string' && !empty($tokens)) {
                    for ($i=0, $len=count($tokens); $i<$len; $i++) {
                        if (!is_array($tokens[$i]) && preg_match('/^:(localpart|domain|all|user|detail)$/i', $tokens[$i])) {
                            $test['part'] = strtolower(substr($tokens[$i], 1));
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'body':
                $test = array('test' => 'body', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:(raw|content|text)$/i', $tokens[$i])) {
                        $test['part'] = strtolower(substr($tokens[$i], 1));

                        if ($test['part'] == 'content') {
                            $test['content'] = $tokens[++$i];
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'date':
            case 'currentdate':
                $test = array('test' => $token, 'not' => $not);

                $test['arg']  = array_pop($tokens);
                $test['part'] = array_pop($tokens);

                if ($token == 'date') {
                    $test['header']  = array_pop($tokens);
                }

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:zone$/i', $tokens[$i])) {
                        $test['zone'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:originalzone$/i', $tokens[$i])) {
                        $test['originalzone'] = true;
                    }
                }

                $tests[] = $test;
                break;

            case 'duplicate':
                $test = array('test' => $token, 'not' => $not);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])) {
                        if (preg_match('/^:(handle|header|uniqueid|seconds)$/i', $tokens[$i], $m)) {
                            $test[strtolower($m[1])] = $tokens[++$i];
                        }
                        else if (preg_match('/^:last$/i', $tokens[$i])) {
                            $test['last'] = true;
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'exists':
                $tests[] = array('test' => 'exists', 'not' => $not,
                    'arg'  => array_pop($tokens));
                break;

            case 'true':
                $tests[] = array('test' => 'true', 'not' => $not);
                break;

            case 'false':
                $tests[] = array('test' => 'true', 'not' => !$not);
                break;
            }

            // goto actions...
            if ($separator == '{') {
                break;
            }
        }

        // ...and actions block
        $actions = $this->_parse_actions($content, $position);

        if ($tests && $actions) {
            $result = array(
                'type'     => $cond,
                'tests'    => $tests,
                'actions'  => $actions,
                'join'     => $join,
                'disabled' => $disabled,
            );
        }

        return $result;
    }
210247
@Override
  public List<String> tokenize(final String text) {
    final List<String> l = new ArrayList<>();
    final StringTokenizer st = new StringTokenizer(text, nlTokenizingChars, true);
    while (st.hasMoreElements()) {
      String token = st.nextToken();
      String origToken = token;
      if (token.length() > 1) {
        if (startsWithQuote(token) && endsWithQuote(token) && token.length() > 2) {
          l.add(token.substring(0, 1));
          l.add(token.substring(1, token.length()-1));
          l.add(token.substring(token.length()-1, token.length()));
        } else if (endsWithQuote(token)) {
          int cnt = 0;
          while (endsWithQuote(token)) {
            token = token.substring(0, token.length() - 1);
            cnt++;
          }
          l.add(token);
          for (int i = origToken.length() - cnt; i < origToken.length(); i++) {
            l.add(origToken.substring(i, i + 1));
          }
        } else if (startsWithQuote(token)) {
          while (startsWithQuote(token)) {
            l.add(token.substring(0, 1));
            token = token.substring(1, token.length());
          }
          l.add(token);
        } else {
          l.add(token);
        }
      } else {
        l.add(token);
      }
    }
    return joinEMailsAndUrls(l);
  }
94735
private int tokenize(StringBuilder sb, String path, int index){
        String[] tokens = path.split("/");
        for (String s: tokens) {
            if (s.length()==0)
                continue;
            sb.append("name").append(index++)
                    .append("=").append(s).append(",");
        }
        return index;
    }
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char