Tokenization rules
480754
public function setupRules()
    {
        $this->rules->addMany([
            'keyword'     => new Rule(new WordMatcher($this->_keywords, ['escape' => false])),
            'symbol.type' => new Rule(new WordMatcher($this->_types, ['escape' => false])),

            'constant' => new Rule(new WordMatcher(['FALSE', 'TRUE', 'NULL'])),
            'comment'  => new Rule(new CommentMatcher(['#', '--'], [['/*', '*/']])),
            
            'string'   => CommonFeatures::strings(['single' => '\'', 'double' => '"']),

            'number'          => new Rule(new RegexMatcher('/\b(-?\d+)\b/i')),
            'call'            => new Rule(new RegexMatcher('/([a-z_]\w*)\s*\(/i'), ['priority' => -1]),
            'operator.escape' => new Rule(new RegexMatcher('/(\\[\\0\'|bnrtZ%_])/'), ['context' => ['string']])
        ]);
    }
375586
function sanitizeWord (token, word, rules, uncountables) {
  let sanitized = word
  // Empty string or doesn't need fixing.
  if (!token.length || uncountables.hasOwnProperty(token)) {
    return sanitized
  }

  for (const rule of [ ...rules, ].reverse()) {
    if (rule[0].test(word)) {
      sanitized = replace(word, rule)
      break
    }
  }

  return sanitized
}
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
717836
def t_REGULATION(self, token):
        '
        indents = token.lexer.lexmatch.group("indents")
        indents = len(indents)/4 if indents else 0
        reg = token.lexer.lexmatch.group("reg").decode("utf8")
        text = token.lexer.lexmatch.group("text").decode("utf8")
        token.value = (indents, reg, text)
        token.lexer.lineno += 1
        return token
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
302865
function Tokenize(query) {
  var parts = __splitTrim(query, WHERE);
  var pathway = parts[0];
  var where = parts[1];

  synopsis.pathway = __splitTrim(pathway, COMMA);
  for (var i = 0, len = synopsis.pathway.length; i < len; i++) {
    synopsis.pathway[i] = __splitTrim(synopsis.pathway[i], DELIMITER);
    if (synopsis.pathway[i][0] == WILDCARD)
      synopsis.pathway[i].shift();
    if (synopsis.pathway[i].length === 0)
      synopsis.pathway.splice(i, 1);
  }

  var lastLeft = -1,
    lastRight = -1,
    current = 0;
  while (current < where.length) {
    if (where[current] === LEFT) {
      lastLeft = current;
    } else if (where[current] === RIGHT) {
      lastRight = current;
      if (lastRight > lastLeft && lastLeft !== -1) {
        var k = 'gr' + '_' + new Date().getTime();
        synopsis.groups[k] = where.substring(lastLeft + 1, lastRight);
        where = where.replace(LEFT + synopsis.groups[k] + RIGHT, k);
        current = -1;
      }
    }
    current += 1;
  }
  LogicalGrouping(AST, where);
}
826595
def _generate_tokens(pat: GenericAny, text: str) -> Iterator[Token]:
    
    rules = _TYPE_RULES
    keys = _TYPE_KEYS
    groupindex = pat.groupindex
    scanner = pat.scanner(text)
    for m in iter(scanner.match, None):
        lastgroup = m.lastgroup
        func = rules[keys[groupindex[lastgroup] - 1]]
        if func is not None:
            yield func(m.group(lastgroup))
1287959
private void checkTokenized() {
        if (tokens == null) {
            if (chars == null) {
                // still call tokenize as subclass may do some work
                final List<String> split = tokenize(null, 0, 0);
                tokens = split.toArray(new String[split.size()]);
            } else {
                final List<String> split = tokenize(chars, 0, chars.length);
                tokens = split.toArray(new String[split.size()]);
            }
        }
    }
670192
def __add_tokenization(self, tree):
        """"""
        for token_id in self.get_token_ids(tree):
            self.add_node(token_id, layers={self.ns})
            self.tokens.append(token_id)
489251
public function setupRules()
    {
        $identifier = '-?[_a-zA-Z]+[_a-zA-Z0-9-]*';
        $at = [
            'charset', 'import', 'namespace',
            'media', 'supports', 'document', 'page', 'font-face', 'keyframes', 'viewport', 'counter-style',
            'font-feature-values', 'swash', 'ornaments', 'annotation', 'stylistic', 'styleset', 'character-variant'
        ];
        
        $this->rules->addMany([
            'string' => CommonFeatures::strings(['single' => '\'', 'double' => '"'], [
                'context' => $this->everywhere()
            ]),

            'meta.declaration' => [
                new OpenRule(new SubStringMatcher('{'), [
                    'context' => ['!meta.declaration.media', '!comment'],
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
                new CloseRule(new SubStringMatcher('}')),
            ],

            'meta.declaration.media' => [
                new Rule(new RegexMatcher('/@media(.*?\{)/'), [
                    'context' => Validator::everywhere(),
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
            ],

            'meta.declaration.rule' => [
                new OpenRule(new RegexMatcher('/@media.*(\()/'), [
                    'context' => ['meta.declaration.media'],
                    'factory' => new TokenFactory(MetaToken::class)
                ]),
                new CloseRule(new SubStringMatcher(')')),
            ],

            'keyword.at-rule' => new Rule(new RegexMatcher('/(@(?:-[a-z]+-)?(?:'.implode('|', $at).'))/'), [
                'priority' => 2
            ]),

            'symbol.selector.id'    => new Rule(new RegexMatcher("/(#$identifier)/i")),
            'symbol.selector.tag'   => new Rule(new RegexMatcher('/(?>[\s}]|^)(?=(\w+)[^;]*\{)/ms')),
            'symbol.selector.class' => new Rule(new RegexMatcher("/(\\.$identifier)/i")),

            'symbol.selector.class.pseudo' => new Rule(new RegexMatcher("/(:{1,2}$identifier)/")),

            'number' => new Rule(new RegexMatcher("/([-+]?[0-9]*\\.?[0-9]+([\\w%]+)?)/"), [
                'context'  => ['meta.declaration', '!constant.color', '!comment', '!symbol', '!comment', '!string'],
                'priority' => 0
            ]),
            
            'symbol.property' => new Rule(new RegexMatcher("/($identifier:)/"), [
                'context' => ['meta.declaration', '!symbol', '!comment'],
                'priority' => 2
            ]),

            'call' => new Rule(new RegexMatcher("/($identifier)\\s*\\((?:(?P<string>[a-z].*?)|.*?)\\)/", [
                1 => Token::NAME,
                'string' => 'string.argument'
            ]), [
                'context' => ['meta.declaration', '!comment', '!string', '!keyword']
            ]),

            'constant.color' => [
                new Rule(new RegexMatcher("/(#[0-9a-f]{3,6})/i"), [
                    'priority' => 2,
                    'context'  => ['meta.declaration', '!comment']
                ]),
                new Rule(new WordMatcher([
                    'white', 'silver', 'gray', 'black', 'red', 'maroon', 'yellow', 'olive',
                    'lime', 'green', 'aqua', 'teal', 'blue', 'navy', 'fuchsia', 'purple'
                ]), [
                    'context'  => ['meta.declaration', '!comment', '!symbol', '!variable']
                ]),
            ],

            'operator' => new Rule(new WordMatcher(['>', '+', '*', '!important'], ['separated' => false]), [
                'context' => $this->everywhere()
            ]),

            'operator.punctuation' => new Rule(new SubStringMatcher(';'), [
                'context' => $this->everywhere()
            ]),

            'comment' => new Rule(new CommentMatcher([], [['/*', '*/']]), ['context' => $this->everywhere()])
        ]);
    }
390902
def _tokenize(self, text):
        """"""
        text = self._clean_text(text)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        text = self._tokenize_chinese_chars(text)
        orig_tokens = self._whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if self.lower:
                token = token.lower()
                token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token))

        output_tokens = self._whitespace_tokenize(' '.join(split_tokens))
        return output_tokens
447566
function processUnary(tokenizer, scope) {
  const unaryAddSubtract = {
    '-': value => -value,
    '+': value => value
  };

  if (tokenizer.token.match(types.ADD_SUBTRACT)) {
    return createOperatorNode(unaryAddSubtract[tokenizer.token.lexeme], [processUnary(tokenizer.skipToken(), scope)]);
  }

  if (tokenizer.token.match(types.INCREASE_DECREASE) || tokenizer.token.match(types.LOGICAL_NEGATION)) {
    return createOperatorNode(tokenizer.token.value, [processUnary(tokenizer.skipToken(), scope)]);
  }

  return processIdentifiers(tokenizer, scope);
}
372458
function () {
            if (vm.inputText == "" || tokenizer == null) {
                vm.tokens = [];
                // lattice = null;
                return;
            }
            try {
                // lattice = tokenizer.getLattice(vm.inputText);
                vm.tokens = tokenizer.tokenize(vm.inputText);
            } catch (e) {
                console.log(e);
                // lattice = null;
                vm.tokens = [];
            }
        }
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
84369
def tokenize_init(spec):
    

    tokens = [Token('__SOF__', '__SOF__', 0)]
    re_token = '|'.join([
        '(?P<{}>{})'.format(name, regex) for name, regex in spec
    ])

    return tokens, re_token
899146
def tokenize(self, s):
    """"""
    s = tf.compat.as_text(s)

    if self.reserved_tokens:
      # First split out the reserved tokens
      substrs = self._reserved_tokens_re.split(s)
    else:
      substrs = [s]

    toks = []
    for substr in substrs:
      if substr in self.reserved_tokens:
        toks.append(substr)
      else:
        toks.extend(self._alphanum_re.split(substr))

    # Filter out empty strings
    toks = [t for t in toks if t]
    return toks