Return a list of tokens, using ``tokenizer``.

        :param tokenizer: (optional) A tokenizer object. If None, defaults to
            this blob's default tokenizer.
428927
def tokenize(self, tokenizer=None):
        
        t = tokenizer if tokenizer is not None else self.tokenizer
        return WordList(t.tokenize(self.raw))
609553
function parseItemOrGroup(tokenizer) {
  var tkn = tokenizer.next();
  if (!tkn) return null;
  if (tkn.typ == '{') {
    return parseGroup(tokenizer);
  }
  return {tag: tkn.typ, children: tkn.txt};
}
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
681028
def get_prompt_tokens(self, _):
        
        if self.shell_ctx.default_command:
            prompt = 'az {}>> '.format(self.shell_ctx.default_command)
        else:
            prompt = 'az>> '
        return [(Token.Az, prompt)]
854859
function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
496079
def tokenize(self, token_list):
    """"""
    return [
        self._vocab_dict.get(token, self._vocab_dict[self.UNK])
        for token in token_list
    ]
464289
def process_tokens(
    tokens: Sequence[Token], *, end: str = "\n", sep: str = " "
) -> Tuple[str, str]:
    
    # Flatten the list of tokens in case some of them are of
    # class UnicodeSequence:
    flat_tokens = list()  # type: List[Token]
    for token in tokens:
        if isinstance(token, UnicodeSequence):
            flat_tokens.extend(token.tuple())
        else:
            flat_tokens.append(token)

    with_color = _process_tokens(flat_tokens, end=end, sep=sep, color=True)
    without_color = _process_tokens(flat_tokens, end=end, sep=sep, color=False)
    return (with_color, without_color)
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens
276293
public List<T> tokenize() {
    // System.out.println("tokenize called");
    List<T> result = new ArrayList<T>();
    while (hasNext()) {
      result.add(next());
    }
    return result;
  }
891575
def _consume_python_token(self, token):
        
        ret = []
        line, col = token[3]
        if line > 1:
            ret = self._buffer[:line-1]
            self._buffer[:line-1] = []
        ret.append(self._buffer[0][:col])
        self._buffer[0] = self._buffer[0][col:]
        return ''.join(ret)
1045264
def get_tokens(max_value):
  
  vocab = [str(i) for i in range(max_value)]
  vocab = set(vocab)
  vocab.update(CodeOp.LITERALS)
  vocab.update(CodeOp.KEYWORDS)
  vocab |= set("".join(vocab))
  return sorted(vocab)
364563
def tokenize(cls, obj):
    
    tokens = {}

    try:
      token_iterator = cls.make_iterable(obj)
      _lang = cls.language_definition()
      tokens = {k: [] for k in _lang.argument_types}

      prev, current = None, next(token_iterator)

      while True:
        token = [None, None]
        arg_type = None

        for arg_type in _lang.argument_types:
          arg_start_seq = _lang.argument_types[arg_type]
          arg_delimiters = _lang.value_delimiters[arg_type]

          if prev is None and arg_start_seq is None:  # just start to scan and got "default" token
            token[1] = current
            break
          elif arg_start_seq is None and prev[0] is None:  # next default token
            token[1] = current
            break
          elif arg_start_seq is None and prev[0] is not None:  # default token used after non-default tokens
            token[1] = current
            break
          elif arg_start_seq is None:  # value of non-default token
            prev[1] = current
            token = None
            break
          elif current[:len(arg_start_seq)] == arg_start_seq:
            token[0] = current[len(arg_start_seq):]

            for delimiter in arg_delimiters:
              if delimiter == " ":
                continue

              if delimiter in token[0]:
                _delim = str(token[0]).partition(delimiter)
                token = [_delim[0], _delim[2]]
                break
            break

        if token:
          tokens[arg_type].append(token)

        prev, current = token, next(token_iterator)
    except StopIteration:
      return tokens
741807
def tokens(self):
    
    seq = self.word_tokenizer.transform(Sequence(self.raw))
    return WordList(seq.tokens(), parent=self, language=self.language.code)
1197908
private void readTillEOL(StreamTokenizer tokenizer) throws IOException {
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
        }

        tokenizer.pushBack();
    }
1205305
def split_token_in_parts(token):
    
    result = []
    current = []
    for part in token + (':', ):
        if part == ':':
            if current:
                result.append(tuple(current))
                current = []
        else:
            current.append(part)

    return result
449931
public function getDocumentsForTokenizer(Tokenizer $tokenizer)
    {
        $this->tokenizer = $tokenizer;
        $resultsPerTokens = $this->queryTokens();

        $flatResults = $this->flattenResults($resultsPerTokens);
        $flatResults = $this->applyQueryCoordination($flatResults);
        usort($flatResults, array($this, 'scoreCompare'));

        $flatResults = array_merge($this->getSearchSuggestions(), $flatResults);

        return $flatResults;
    }