Do not produce empty tokens.
963409
def something(TokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if not text:
            return
        yield match.start(), TokenClass, text
        context.pos = match.end()
    return callback
734043
def contains(self, token: str) -> bool:
        """"""
        self._validate_token(token)
        return token in self
1141492
def detect(self, tokens):
        """"""
        results = []
        for abbr_span, long_span in self.detect_spans(tokens):
            results.append((tokens[abbr_span[0]:abbr_span[1]], tokens[long_span[0]:long_span[1]]))
        return results
1362214
private static boolean areTokensValid(Token... tokens)
    {
        for (Token token : tokens)
        {
            if (!isTokenValid(token))
                return false;
        }
        return true;
    }
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
568230
private boolean isAtEOL() {
		int next = skipLineSpace(index);
		return next >= tokens.size() || tokens.get(next).kind == NewLine;
	}
932694
private function popWhitespace()
    {
        while ($this->isToken(reset($this->tokens), Tokenizer::TOKEN_WHITESPACE)) {
            array_shift($this->tokens);
        }
    }
385573
def tokens(cmd, text)
      valide_command!(cmd)
      Enumerator.new do |yielder|
        call(cmd, text).each do |freeling_line|
          yielder << parse_token_line(freeling_line) unless freeling_line.empty?
        end
      end
    end
1340700
def split_tokens(self):
        """"""
        if not self._split_tokens_calculated:
            # split into items (whitespace split)
            self._split_tokens = self._line_str.split()
            self._split_tokens_calculated = True

        return self._split_tokens
291002
function() {
    var captures;
    if (captures = /^u\+[0-9a-f?]{1,6}(?:-[0-9a-f]{1,6})?/i.exec(this.str)) {
      this.skip(captures);
      return new Token('literal', new nodes.Literal(captures[0]));
    }
  }
169246
function() {
    var captures;
    if (captures = /^([.]{1,3}|&&|\|\||[!<>=?:]=|\*\*|[-+*\/%]=?|[,=?:!~<>&\[\]])([ \t]*)/.exec(this.str)) {
      var op = captures[1];
      this.skip(captures);
      op = alias[op] || op;
      var tok = new Token(op, op);
      tok.space = captures[2];
      this.isURL = false;
      return tok;
    }
  }
476277
protected function filteredTokens()
    {
        foreach ($this->tokens as $token) {
            if ($token->reloadRequired() || $token->reloadRequiredIfError()) {
                yield $token;
            }
        }
    }
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
1163944
protected function skipWhitespace()
	{
		while (++$this->i < $this->cnt && $this->tokens[$this->i][0] === T_WHITESPACE);
	}
984663
private static boolean checkTokenValid(final char[] token) {
        if (token == null || token.length == 0) {
            return true;
        }
        for (int i = 0; i < token.length; i++) {
            if (token[i] == '\n') {
                return false;
            }

        }
        return true;
    }
145058
def _flatten_up_to_token(self, token):
        """"""
        if token.is_group:
            token = next(token.flatten())

        for t in self._curr_stmt.flatten():
            if t == token:
                break
            yield t