Tokenization rules definition
336604
public function setupRules()
    {
        $identifier = '\w+';
        $number = '[+-]?(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?';

        $this->rules->addMany([
            'string'  => CommonFeatures::strings(['single' => '\'', 'double' => '"'], [
                'context' => ['!keyword', '!comment', '!string', '!language', '!number'],
            ]),

            'comment' => new Rule(new CommentMatcher(['#'])),

            'keyword' => new Rule(new WordMatcher([
                'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach',
                'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then',
                'unless', 'until', 'while', 'use', 'print', 'new', 'BEGIN',
                'sub', 'CHECK', 'INIT', 'END', 'return', 'exit'
            ])),

            'operator.escape' => new Rule(new RegexMatcher('/(\\\.)/'), [
                'context' => ['string']
            ]),

            'string.nowdoc'  => new Rule(
                new RegexMatcher('/<<\s*\'(\w+)\';(?P<string>.*?)\R\1/sm', [
                    'string' => Token::NAME,
                          0  => 'keyword.nowdoc'
                ]), ['context' => ['!comment']]
            ),

            'language.shell' => new Rule(new SubStringMatcher('`'), [
                'context' => ['!operator.escape', '!comment', '!string', '!keyword.nowdoc'],
                'factory' => new TokenFactory(ContextualToken::class),
            ]),

            'variable.scalar' => new Rule(new RegexMatcher("/(\\\$$identifier)/")),
            'variable.array'  => new Rule(new RegexMatcher("/(\\@$identifier)/")),
            'variable.hash'   => new Rule(new RegexMatcher("/(\\%$identifier)/")),

            'variable.property'   => new Rule(new RegexMatcher("/\\\$$identifier{($identifier)}/")),

            // Stupidly named var? Perl one, for sure.
            'variable.special'   => new Rule(new RegexMatcher('/([$@%][^\s\w]+[\w]*)/')),

            'operator' => [
                new Rule(new RegexMatcher('/(-[rwxoRWXOezsfdlpSbctugkTBMAC])/')),
                new Rule(new WordMatcher([
                    'not', 'and', 'or', 'xor', 'goto', 'last', 'next', 'redo', 'dump',
                    'eq', 'ne', 'cmp', 'not', 'and', 'or', 'xor'
                ], ['atomic' => true])),
            ],

            'call' => new Rule(new RegexMatcher('/([a-z]\w+)(?:\s*\(|\s+[$%@"\'`{])/i')),

            'number' => [
                new Rule(new RegexMatcher("/(\\b|\"|')$number\\1/", [
                    0 => Token::NAME
                ]), ['priority' => 5]),
            ],

            'string.regex' => [
                new OpenRule(new RegexMatcher('#~\s*[ms]?(/).*?/#m'), [
                    'context' => Validator::everywhere()
                ]),
                new OpenRule(new RegexMatcher('#~\s*(s/).*?/#m')),

                new Rule(new RegexMatcher('#(?=\/.*?(/[gimuy]{0,5}))#m'), [
                    'priority' => 1,
                    'factory'  => new TokenFactory(ContextualToken::class),
                    'context'  => ['!operator.escape', 'string.regex']
                ])
            ],
            
            'symbol.iterator' => [
                new Rule(new RegexMatcher('#(<\w+>)#s'))
            ]
        ]);
    }
341388
final boolean tokenIs(String s)
  {
    return (m_token != null) ? (m_token.equals(s)) : (s == null);
  }
1263263
function tokenizeSentences(tokenize, opts) {
    opts = _.defaults(opts || {}, {
        newlineBoundary: true
    });

    var splitRe = /\s/;

    return tokenize.serie(
        // Split into words
        tokenize.re(splitRe, { split: true }),

        // Merge words as sentences
        tokenize.splitAndMerge(function(word, token, prev, next, i, tokens) {
            var tmp;
            var endOfSentence = [word, null];
            var sentenceNotOver = word;

            // Find the next word
            var nextWord = _.find(tokens.slice(i+1), function(tok) {
                return !splitRe.test(tok.value)
            });

            // Newline boundaries
            if (word === '\n' && opts.newlineBoundary) return endOfSentence;

            if (isBoundaryChar(word) ||
                utils.endsWithChar(word, "?!"))
            {
                return endOfSentence;
            }

            // A dot might indicate the end sentences
            // Exception: The next sentence starts with a word (non abbreviation)
            //            that has a capital letter.
            if (utils.endsWithChar(word, '.')) {
                // Check if there is a next word
                if (nextWord) {
                    // This should be improved with machine learning

                    // Single character abbr.
                    if (word.length === 2 && isNaN(word.charAt(0)) && word.match(/[a-zA-Z]/)) {
                        return sentenceNotOver;
                    }

                    // Common abbr. that often do not end sentences
                    if (isCommonAbbreviation(word)) {
                        return sentenceNotOver;
                    }

                    // Next word starts with capital word, but current sentence is
                    // quite short
                    if (isSentenceStarter(nextWord.value)) {
                        if (isTimeAbbreviation(word, nextWord.value)) {
                            return sentenceNotOver;
                        }

                        // Dealing with names at the start of sentences
                        /*if (isNameAbbreviation(wordCount, words.slice(i, 6))) {
                            return word;
                        }*/

                        if (isNumber(nextWord.value) && isCustomAbbreviation(word)) {
                            return sentenceNotOver;
                        }
                    }
                    else {
                        // Skip ellipsis
                        if (utils.endsWith(word, "..")) {
                            return sentenceNotOver;
                        }

                        //// Skip abbreviations
                        // Short words + dot or a dot after each letter
                        if (isDottedAbbreviation(word) || isCustomAbbreviation(word)) {
                            return sentenceNotOver;
                        }
                    }
                }

                return endOfSentence;
            }

            // Check if the word has a dot in it
            if ((index = word.indexOf(".")) > -1) {
                if (isNumber(word, index)) {
                    return sentenceNotOver;
                }

                // Custom dotted abbreviations (like K.L.M or I.C.T)
                if (isDottedAbbreviation(word)) {
                    return sentenceNotOver;
                }

                // Skip urls / emails and the like
                if (isURL(word) || isPhoneNr(word)) {
                    return sentenceNotOver;
                }
            }

            if (tmp = isConcatenated(word)) {
                return [
                    tmp[0]+tmp[1], null, tmp[2]
                ];
            }

            return sentenceNotOver;
        }),

        // Filter empty sentences
        tokenize.filter(function(sentence) {
            if (sentence.trim() == '') return false;
            return true;
        })
    );
}
366976
private void parseRule( Rule rule ) {
        ruleStack.add( rule );
        for( ;; ) {
            int ch = reader.nextBlockMarker();
            switch( ch ) {
                case -1:
                    throw createException( "Unexpected end of Less data" );
                case ';':
                    parseSemicolon( rule );
                    break;
                case '{':
                    parseBlock( rule );
                    break;
                case '}':
                    parseSemicolon( rule );
                    ruleStack.removeLast();
                    return;
                default:
                    throw createException( "Unrecognized input: '" + reader.getLookAhead() + "'" );
            }
        }

    }
441680
def tokenize(self):

        

        self.tokens = []
        self.terms = OrderedDict()

        # Generate tokens.
        for token in utils.tokenize(self.text):

            # Ignore stopwords.
            if token['unstemmed'] in self.stopwords:
                self.tokens.append(None)

            else:

                # Token:
                self.tokens.append(token)

                # Term:
                offsets = self.terms.setdefault(token['stemmed'], [])
                offsets.append(token['offset'])
363660
function tokenize(str, wordRegexDelimiters, phraseRegexDelimiters) {
    if (wordRegexDelimiters === void 0) { wordRegexDelimiters = ' '; }
    if (phraseRegexDelimiters === void 0) { phraseRegexDelimiters = ''; }
    /* tslint:enable */
    var regexStr = '(?:[' + phraseRegexDelimiters + '])([^' + phraseRegexDelimiters + ']+)(?:[' + phraseRegexDelimiters + '])|([^' + wordRegexDelimiters + ']+)';
    var preTokenized = str.split(new RegExp(regexStr, 'g'));
    var result = [];
    var preTokenizedLength = preTokenized.length;
    var token;
    var replacePhraseDelimiters = new RegExp('[' + phraseRegexDelimiters + ']+', 'g');
    for (var i = 0; i < preTokenizedLength; i += 1) {
        token = preTokenized[i];
        if (token && token.length && token !== wordRegexDelimiters) {
            result.push(token.replace(replacePhraseDelimiters, ''));
        }
    }
    return result;
}
129790
def tokenizer(self):
        
        domain = 0
        if domain not in self.domains:
            self.register_domain(domain=domain)
        return self.domains[domain].tokenizer
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
1065304
def tokenize(text, format=None):
    
    text = Text(text)
    text = text.replace("\t", " ")
    tokens = re.findall(patterns, text)
    tokens = [token[0] for token in tokens]
    if format == "text":
        return " ".join(tokens)
    else:
        return tokens
458115
public final void ruleAbstractToken() throws RecognitionException {

        		int stackSize = keepStackSize();
        	
        try {
            // InternalXtext.g:433:2: ( ( ( rule__AbstractToken__Alternatives ) ) )
            // InternalXtext.g:434:2: ( ( rule__AbstractToken__Alternatives ) )
            {
            // InternalXtext.g:434:2: ( ( rule__AbstractToken__Alternatives ) )
            // InternalXtext.g:435:3: ( rule__AbstractToken__Alternatives )
            {
             before(grammarAccess.getAbstractTokenAccess().getAlternatives()); 
            // InternalXtext.g:436:3: ( rule__AbstractToken__Alternatives )
            // InternalXtext.g:436:4: rule__AbstractToken__Alternatives
            {
            pushFollow(FollowSets000.FOLLOW_2);
            rule__AbstractToken__Alternatives();

            state._fsp--;


            }

             after(grammarAccess.getAbstractTokenAccess().getAlternatives()); 

            }


            }

        }
        catch (RecognitionException re) {
            reportError(re);
            recover(input,re);
        }
        finally {

            	restoreStackSize(stackSize);

        }
        return ;
    }
1008273
private function _tokenize_rule($content, &$position)
    {
        $cond = strtolower(self::tokenize($content, 1, $position));
        if ($cond != 'if' && $cond != 'elsif' && $cond != 'else') {
            return null;
        }

        $disabled = false;
        $join     = false;
        $join_not = false;
        $length   = strlen($content);

        // disabled rule (false + comment): if false # .....
        if (preg_match('/^\s*false\s+#\s*/i', substr($content, $position, 20), $m)) {
            $position += strlen($m[0]);
            $disabled = true;
        }

        while ($position < $length) {
            $tokens    = self::tokenize($content, true, $position);
            $separator = array_pop($tokens);

            if (!empty($tokens)) {
                $token = array_shift($tokens);
            }
            else {
                $token = $separator;
            }

            $token = strtolower($token);

            if ($token == 'not') {
                $not = true;
                $token = strtolower(array_shift($tokens));
            }
            else {
                $not = false;
            }

            // we support "not allof" as a negation of allof sub-tests
            if ($join_not) {
                $not = !$not;
            }

            switch ($token) {
            case 'allof':
                $join     = true;
                $join_not = $not;
                break;

            case 'anyof':
                break;

            case 'size':
                $test = array('test' => 'size', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])
                        && preg_match('/^:(under|over)$/i', $tokens[$i])
                    ) {
                        $test['type'] = strtolower(substr($tokens[$i], 1));
                    }
                }

                $tests[] = $test;
                break;

            case 'header':
            case 'string':
            case 'address':
            case 'envelope':
                $test = array('test' => $token, 'not' => $not);

                $test['arg2'] = array_pop($tokens);
                $test['arg1'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                if ($token != 'header' && $token != 'string' && !empty($tokens)) {
                    for ($i=0, $len=count($tokens); $i<$len; $i++) {
                        if (!is_array($tokens[$i]) && preg_match('/^:(localpart|domain|all|user|detail)$/i', $tokens[$i])) {
                            $test['part'] = strtolower(substr($tokens[$i], 1));
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'body':
                $test = array('test' => 'body', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:(raw|content|text)$/i', $tokens[$i])) {
                        $test['part'] = strtolower(substr($tokens[$i], 1));

                        if ($test['part'] == 'content') {
                            $test['content'] = $tokens[++$i];
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'date':
            case 'currentdate':
                $test = array('test' => $token, 'not' => $not);

                $test['arg']  = array_pop($tokens);
                $test['part'] = array_pop($tokens);

                if ($token == 'date') {
                    $test['header']  = array_pop($tokens);
                }

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:zone$/i', $tokens[$i])) {
                        $test['zone'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:originalzone$/i', $tokens[$i])) {
                        $test['originalzone'] = true;
                    }
                }

                $tests[] = $test;
                break;

            case 'duplicate':
                $test = array('test' => $token, 'not' => $not);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])) {
                        if (preg_match('/^:(handle|header|uniqueid|seconds)$/i', $tokens[$i], $m)) {
                            $test[strtolower($m[1])] = $tokens[++$i];
                        }
                        else if (preg_match('/^:last$/i', $tokens[$i])) {
                            $test['last'] = true;
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'exists':
                $tests[] = array('test' => 'exists', 'not' => $not,
                    'arg'  => array_pop($tokens));
                break;

            case 'true':
                $tests[] = array('test' => 'true', 'not' => $not);
                break;

            case 'false':
                $tests[] = array('test' => 'true', 'not' => !$not);
                break;
            }

            // goto actions...
            if ($separator == '{') {
                break;
            }
        }

        // ...and actions block
        $actions = $this->_parse_actions($content, $position);

        if ($tests && $actions) {
            $result = array(
                'type'     => $cond,
                'tests'    => $tests,
                'actions'  => $actions,
                'join'     => $join,
                'disabled' => $disabled,
            );
        }

        return $result;
    }
1043559
public final AntlrDatatypeRuleToken ruleOpSingleAssign() throws RecognitionException {
        AntlrDatatypeRuleToken current = new AntlrDatatypeRuleToken();

        Token kw=null;


        	enterRule();

        try {
            // InternalPureXbase.g:919:2: (kw= '=' )
            // InternalPureXbase.g:920:2: kw= '='
            {
            kw=(Token)match(input,20,FOLLOW_2); if (state.failed) return current;
            if ( state.backtracking==0 ) {

              		current.merge(kw);
              		newLeafNode(kw, grammarAccess.getOpSingleAssignAccess().getEqualsSignKeyword());
              	
            }

            }

            if ( state.backtracking==0 ) {

              	leaveRule();

            }
        }

            catch (RecognitionException re) {
                recover(input,re);
                appendSkippedTokens();
            }
        finally {
        }
        return current;
    }
904060
function checkAtruler(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkAtkeyword(i)) i += l;
  else return 0;

  if (l = checkTsets(i)) i += l;

  if (l = checkAtrulers(i)) i += l;
  else return 0;

  return i - start;
}
385573
def tokens(cmd, text)
      valide_command!(cmd)
      Enumerator.new do |yielder|
        call(cmd, text).each do |freeling_line|
          yielder << parse_token_line(freeling_line) unless freeling_line.empty?
        end
      end
    end
1309991
private Set<String> tokenize(String fieldValue) {
        FieldAnalyzer analyzer = FieldAnalyzer.findAnalyzer(m_tableDef, m_fieldName);
        return analyzer.extractTerms(fieldValue);
    }
1287959
private void checkTokenized() {
        if (tokens == null) {
            if (chars == null) {
                // still call tokenize as subclass may do some work
                final List<String> split = tokenize(null, 0, 0);
                tokens = split.toArray(new String[split.size()]);
            } else {
                final List<String> split = tokenize(chars, 0, chars.length);
                tokens = split.toArray(new String[split.size()]);
            }
        }
    }
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}