Indexes unknown and reserved tokens.
494777
def _index_unknown_and_reserved_tokens(self, unknown_token, reserved_tokens):
        """"""

        self._unknown_token = unknown_token
        # Thus, constants.UNKNOWN_IDX must be 0.
        self._idx_to_token = [unknown_token]

        if reserved_tokens is None:
            self._reserved_tokens = None
        else:
            self._reserved_tokens = reserved_tokens[:]
            self._idx_to_token.extend(reserved_tokens)

        self._token_to_idx = {token: idx for idx, token in enumerate(self._idx_to_token)}
1053497
def _parse_next_token(self):
        """"""
        while self._position < self.limit:
            token = self._next_pattern()
            if token:
                return token

        return None
568636
def list_tokens(opts):
    '''
    
    '''
    ret = []
    for (dirpath, dirnames, filenames) in salt.utils.path.os_walk(opts['token_dir']):
        for token in filenames:
            ret.append(token)
    return ret
173848
def load(cls, serialized_index):
        """"""
        from lunr import __TARGET_JS_VERSION__

        if isinstance(serialized_index, basestring):
            serialized_index = json.loads(serialized_index)

        if serialized_index["version"] != __TARGET_JS_VERSION__:
            logger.warning(
                "Version mismatch when loading serialized index. "
                "Current version of lunr {} does not match that of serialized "
                "index {}".format(__TARGET_JS_VERSION__, serialized_index["version"])
            )

        field_vectors = {
            ref: Vector(elements) for ref, elements in serialized_index["fieldVectors"]
        }

        tokenset_builder = TokenSetBuilder()
        inverted_index = {}
        for term, posting in serialized_index["invertedIndex"]:
            tokenset_builder.insert(term)
            inverted_index[term] = posting

        tokenset_builder.finish()

        return Index(
            fields=serialized_index["fields"],
            field_vectors=field_vectors,
            inverted_index=inverted_index,
            token_set=tokenset_builder.root,
            pipeline=Pipeline.load(serialized_index["pipeline"]),
        )
766444
def t_ANY_ID(self, token):
        '
        if token.value in self.KEYWORDS:
            if (token.value == 'annotation_type') and self.cur_indent:
                # annotation_type was added as a reserved keyword relatively
                # late, when there could be identifers with the same name
                # in existing specs. because annotation_type-the-keyword can
                # only be used at the beginning of a non-indented line, this
                # check lets both the keyword and the identifer coexist and
                # maintains backward compatibility.
                # Note: this is kind of a hack, and we should get rid of it if
                # the lexer gets better at telling keywords from identifiers in general.
                return token
            token.type = self.RESERVED.get(token.value, 'KEYWORD')
            return token
        else:
            return token
1285963
def get_word_index(tokens, char_index):
    '''
    
    '''
    for (i, token) in enumerate(tokens):
        if token['char_end'] == 0:
            continue
        if token['char_begin'] <= char_index and char_index <= token['char_end']:
            return i
    return 0
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
657796
public static function initTokenKindToText() {
        self::$tokenKindToText = \array_flip(\array_merge(
            TokenStringMaps::OPERATORS_AND_PUNCTUATORS,
            TokenStringMaps::KEYWORDS,
            TokenStringMaps::RESERVED_WORDS
        ));
    }
60665
def list_tokens(opts):
    '''
    
    '''
    ret = []
    redis_client = _redis_client(opts)
    if not redis_client:
        return []
    serial = salt.payload.Serial(opts)
    try:
        return [k.decode('utf8') for k in redis_client.keys()]
    except Exception as err:
        log.warning('Failed to list keys: %s', err)
        return []
118350
def fetch(index, tokens):
    
    if len(tokens) == 0:
        return set()
    return set.intersection(*[set(index.get(token, [])) for token in tokens])
621512
private function parseTokens()
    {
        $this->tokens = $this->tokenStore->getTokens($this->tokenMask);
        $this->keys = array_keys($this->tokens);
        $this->keyLengths = array_unique(array_map('strlen', $this->keys));
        arsort($this->keyLengths);
        $this->maxLen = reset($this->keyLengths);
    }
784299
def hidden_indices(self):
        """"""
        return tuple(sorted(set(self.micro_indices) -
                            set(self.output_indices)))
744350
def _parse_indices(self):
        """"""
        v_name = self.prior_token
        v_indices = []

        while self.token in (',', '('):
            v_indices.append(self._parse_index(v_name))

        return v_indices
197891
def _iter_tokens(self):
        """"""
        reobj, actions, nextstates = self._rules[self.states[-1]]
        mobj = reobj.match(self.string, self.pos)
        while mobj is not None:
            text = mobj.group(0)
            idx = mobj.lastindex - 1
            nextstate = nextstates[idx]

            # Take action
            actions[idx](self, text)
            while self.tokens:
                yield self.pop_token()
            if nextstate and nextstate != self.states[-1]:
                self.states[-1] = nextstate

            # Update position variables
            self.pos = mobj.end()
            lines = text.split('\n')
            nlines = len(lines) - 1
            if nlines == 0:
                self.offset = self.offset + len(lines[0])
            else:
                self.lineno = self.lineno + nlines
                self.offset = 1 + len(lines[-1])

            reobj, actions, nextstates = self._rules[self.states[-1]]
            mobj = reobj.match(self.string, self.pos)

        if self.pos != len(self.string):
            msg = "unexpected character"
            text = self.string[self.pos]
            raise RunError(msg, self.lineno, self.offset, text)

        yield EndToken("", self.lineno, self.offset)
162014
def _create_indexes(self):
        """"""
        # WARNING: The collection will be locked during the index
        # creation. If the collection has a large number of
        # documents in it, the operation can take a long time.

        # TODO: The creation of indexes can be moved to a Django
        # management command or equivalent. There is also an option to
        # run the indexing on the background, without locking.
        self.collection.ensure_index([('time', pymongo.DESCENDING)])
        self.collection.ensure_index('name')
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
626611
def token_name(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == expected:
        return TokenMatch(None, token.value, (token,))