Lex the input string according to _tsql_lex_re.

    Yields
        (gid, token, line_number)
336642
def _lex(s):
    
    s += '.'  # make sure there's a terminator to know when to stop parsing
    lines = enumerate(s.splitlines(), 1)
    lineno = pos = 0
    try:
        for lineno, line in lines:
            matches = _tsql_lex_re.finditer(line)
            for m in matches:
                gid = m.lastindex
                if gid == 11:
                    raise TSQLSyntaxError('unexpected input',
                                          lineno=lineno,
                                          offset=m.start(),
                                          text=line)
                else:
                    token = m.group(gid)
                    yield (gid, token, lineno)
    except StopIteration:
        pass
1006206
def make_tokenizer(specs):
    """"""

    def compile_spec(spec):
        name, args = spec
        return name, re.compile(*args)

    compiled = [compile_spec(s) for s in specs]

    def match_specs(specs, str, i, position):
        line, pos = position
        for type, regexp in specs:
            m = regexp.match(str, i)
            if m is not None:
                value = m.group()
                nls = value.count(u'\n')
                n_line = line + nls
                if nls == 0:
                    n_pos = pos + len(value)
                else:
                    n_pos = len(value) - value.rfind(u'\n') - 1
                return Token(type, value, (line, pos + 1), (n_line, n_pos))
        else:
            errline = str.splitlines()[line - 1]
            raise LexerError((line, pos + 1), errline)

    def f(str):
        length = len(str)
        line, pos = 1, 0
        i = 0
        while i < length:
            t = match_specs(compiled, str, i, (line, pos))
            yield t
            line, pos = t.end
            i += len(t.value)

    return f
640028
function (match, indexed_rule) {
        var token,
            lines,
            backup;

        if (this.options.backtrack_lexer) {
            // save context
            backup = {
                yylineno: this.yylineno,
                yylloc: {
                    first_line: this.yylloc.first_line,
                    last_line: this.last_line,
                    first_column: this.yylloc.first_column,
                    last_column: this.yylloc.last_column
                },
                yytext: this.yytext,
                match: this.match,
                matches: this.matches,
                matched: this.matched,
                yyleng: this.yyleng,
                offset: this.offset,
                _more: this._more,
                _input: this._input,
                yy: this.yy,
                conditionStack: this.conditionStack.slice(0),
                done: this.done
            };
            if (this.options.ranges) {
                backup.yylloc.range = this.yylloc.range.slice(0);
            }
        }

        lines = match[0].match(/(?:\r\n?|\n).*/g);
        if (lines) {
            this.yylineno += lines.length;
        }
        this.yylloc = {
            first_line: this.yylloc.last_line,
            last_line: this.yylineno + 1,
            first_column: this.yylloc.last_column,
            last_column: lines ?
                         lines[lines.length - 1].length - lines[lines.length - 1].match(/\r?\n?/)[0].length :
                         this.yylloc.last_column + match[0].length
        };
        this.yytext += match[0];
        this.match += match[0];
        this.matches = match;
        this.yyleng = this.yytext.length;
        if (this.options.ranges) {
            this.yylloc.range = [this.offset, this.offset += this.yyleng];
        }
        this._more = false;
        this._backtrack = false;
        this._input = this._input.slice(match[0].length);
        this.matched += match[0];
        token = this.performAction.call(this, this.yy, this, indexed_rule, this.conditionStack[this.conditionStack.length - 1]);
        if (this.done && this._input) {
            this.done = false;
        }
        if (token) {
            return token;
        } else if (this._backtrack) {
            // recover context
            for (var k in backup) {
                this[k] = backup[k];
            }
            return false; // rule action called reject() implying the next rule should be tested instead.
        }
        return false;
    }
758604
def str_lexer(mode):
    
    cast_to_const = ConstStrPool.cast_to_const

    def f_raw(inp_str, pos):
        return cast_to_const(mode) if inp_str.startswith(mode, pos) else None

    def f_collection(inp_str, pos):
        for each in mode:
            if inp_str.startswith(each, pos):
                return cast_to_const(each)
        return None

    if isinstance(mode, str):
        return f_raw

    if len(mode) is 1:
        mode = mode[0]
        return f_raw

    return f_collection
130748
def lex(code, lexer):
    
    try:
        return lexer.get_tokens(code)
    except TypeError as err:
        if (isinstance(err.args[0], str) and
            ('unbound method get_tokens' in err.args[0] or
             'missing 1 required positional argument' in err.args[0])):
            raise TypeError('lex() argument must be a lexer instance, '
                            'not a class')
        raise
578917
function () {
      var property, token;
      for (property in this.lex) {
        token = this.lex[property]

        if (token.hasText) {
          token.compiledRe = this.lex.text.matcher + token.matcher;
        } else {
          token.compiledRe = token.matcher;
        }

        token.compiledRe = new RegExp(token.compiledRe, "gi");
      }
    }
444686
def get_tokens(line: str) -> Iterator[str]:
    
    for token in line.rstrip().split():
        if len(token) > 0:
            yield token
1170033
def _lex(stream):
    
    lines = enumerate(stream, 1)
    line_no = pos = 0
    try:
        while True:
            if pos == 0:
                line_no, line = next(lines)
            matches = _tdl_lex_re.finditer(line, pos)
            pos = 0  # reset; only used for multiline patterns
            for m in matches:
                gid = m.lastindex
                if gid <= 2:  # potentially multiline patterns
                    if gid == 1:  # docstring
                        s, start_line_no, line_no, line, pos = _bounded(
                            '"""', '"""', line, m.end(), line_no, lines)
                    elif gid == 2:  # comment
                        s, start_line_no, line_no, line, pos = _bounded(
                            '#|', '|#', line, m.end(), line_no, lines)
                    yield (gid, s, line_no)
                    break
                elif gid == 30:
                    raise TdlParsingError(
                        ('Syntax error:\n  {}\n {}^'
                         .format(line, ' ' * m.start())),
                        line_number=line_no)
                else:
                    # token = None
                    # if not (6 < gid < 20):
                    #     token = m.group(gid)
                    token = m.group(gid)
                    yield (gid, token, line_no)
    except StopIteration:
        pass
197891
def _iter_tokens(self):
        """"""
        reobj, actions, nextstates = self._rules[self.states[-1]]
        mobj = reobj.match(self.string, self.pos)
        while mobj is not None:
            text = mobj.group(0)
            idx = mobj.lastindex - 1
            nextstate = nextstates[idx]

            # Take action
            actions[idx](self, text)
            while self.tokens:
                yield self.pop_token()
            if nextstate and nextstate != self.states[-1]:
                self.states[-1] = nextstate

            # Update position variables
            self.pos = mobj.end()
            lines = text.split('\n')
            nlines = len(lines) - 1
            if nlines == 0:
                self.offset = self.offset + len(lines[0])
            else:
                self.lineno = self.lineno + nlines
                self.offset = 1 + len(lines[-1])

            reobj, actions, nextstates = self._rules[self.states[-1]]
            mobj = reobj.match(self.string, self.pos)

        if self.pos != len(self.string):
            msg = "unexpected character"
            text = self.string[self.pos]
            raise RunError(msg, self.lineno, self.offset, text)

        yield EndToken("", self.lineno, self.offset)
310273
def eval_string(stri):
    ''
    tokens = shlex.split(stri)

    return run_write_read(['plash', 'eval'], '\n'.join(tokens).encode()).decode()
898694
def find_column(token):
    
    i = token.lexpos
    input = token.lexer.lexdata

    while i > 0:
        if input[i - 1] == '\n':
            break
        i -= 1

    column = token.lexpos - i + 1

    return column
637425
function Lexer(str, options) {
  options = options || {};
  this.stash = [];
  this.indentStack = [];
  this.indentRe = null;
  this.lineno = 1;
  this.column = 1;

  // HACK!
  function comment(str, val, offset, s) {
    var inComment = s.lastIndexOf('/*', offset) > s.lastIndexOf('*/', offset)
      , commentIdx = s.lastIndexOf('//', offset)
      , i = s.lastIndexOf('\n', offset)
      , double = 0
      , single = 0;

    if (~commentIdx && commentIdx > i) {
      while (i != offset) {
        if ("'" == s[i]) single ? single-- : single++;
        if ('"' == s[i]) double ? double-- : double++;

        if ('/' == s[i] && '/' == s[i + 1]) {
          inComment = !single && !double;
          break;
        }
        ++i;
      }
    }

    return inComment
      ? str
      : ((val === ',' && /^[,\t\n]+$/.test(str)) ? str.replace(/\n/, '\r') : val + '\r');
  };

  // Remove UTF-8 BOM.
  if ('\uFEFF' == str.charAt(0)) str = str.slice(1);

  this.str = str
    .replace(/\s+$/, '\n')
    .replace(/\r\n?/g, '\n')
    .replace(/\\ *\n/g, '\r')
    .replace(/([,(:](?!\/\/[^ ])) *(?:\/\/[^\n]*|\/\*.*?\*\/)?\n\s*/g, comment)
    .replace(/\s*\n[ \t]*([,)])/g, comment);
}
1280257
def lex_document(self, cli, document):
        
        # Cache of already lexed lines.
        cache = {}

        # Pygments generators that are currently lexing.
        line_generators = {}  # Map lexer generator to the line number.

        def get_syntax_sync():
            " The Syntax synchronisation objcet that we currently use. "
            if self.sync_from_start(cli):
                return SyncFromStart()
            else:
                return self.syntax_sync

        def find_closest_generator(i):
            " Return a generator close to line 'i', or None if none was fonud. "
            for generator, lineno in line_generators.items():
                if lineno < i and i - lineno < self.REUSE_GENERATOR_MAX_DISTANCE:
                    return generator

        def create_line_generator(start_lineno, column=0):
            """
            Create a generator that yields the lexed lines.
            Each iteration it yields a (line_number, [(token, text), ...]) tuple.
            """
            def get_tokens():
                text = '\n'.join(document.lines[start_lineno:])[column:]

                # We call `get_tokens_unprocessed`, because `get_tokens` will
                # still replace \r\n and \r by \n.  (We don't want that,
                # Pygments should return exactly the same amount of text, as we
                # have given as input.)
                for _, t, v in self.pygments_lexer.get_tokens_unprocessed(text):
                    yield t, v

            return enumerate(split_lines(get_tokens()), start_lineno)

        def get_generator(i):
            """
            Find an already started generator that is close, or create a new one.
            """
            # Find closest line generator.
            generator = find_closest_generator(i)
            if generator:
                return generator

            # No generator found. Determine starting point for the syntax
            # synchronisation first.

            # Go at least x lines back. (Make scrolling upwards more
            # efficient.)
            i = max(0, i - self.MIN_LINES_BACKWARDS)

            if i == 0:
                row = 0
                column = 0
            else:
                row, column = get_syntax_sync().get_sync_start_position(document, i)

            # Find generator close to this point, or otherwise create a new one.
            generator = find_closest_generator(i)
            if generator:
                return generator
            else:
                generator = create_line_generator(row, column)

            # If the column is not 0, ignore the first line. (Which is
            # incomplete. This happens when the synchronisation algorithm tells
            # us to start parsing in the middle of a line.)
            if column:
                next(generator)
                row += 1

            line_generators[generator] = row
            return generator

        def get_line(i):
            " Return the tokens for a given line number. "
            try:
                return cache[i]
            except KeyError:
                generator = get_generator(i)

                # Exhaust the generator, until we find the requested line.
                for num, line in generator:
                    cache[num] = line
                    if num == i:
                        line_generators[generator] = i

                        # Remove the next item from the cache.
                        # (It could happen that it's already there, because of
                        # another generator that started filling these lines,
                        # but we want to synchronise these lines with the
                        # current lexer's state.)
                        if num + 1 in cache:
                            del cache[num + 1]

                        return cache[num]
            return []

        return get_line
309445
def _parse_tokenize(self, rule):
        """"""

        for token in self._TOKENIZE_RE.split(rule):
            # Skip empty tokens
            if not token or token.isspace():
                continue

            # Handle leading parens on the token
            clean = token.lstrip('(')
            for i in range(len(token) - len(clean)):
                yield '(', '('

            # If it was only parentheses, continue
            if not clean:
                continue
            else:
                token = clean

            # Handle trailing parens on the token
            clean = token.rstrip(')')
            trail = len(token) - len(clean)

            # Yield the cleaned token
            lowered = clean.lower()
            if lowered in ('and', 'or', 'not'):
                # Special tokens
                yield lowered, clean
            elif clean:
                # Not a special token, but not composed solely of ')'
                if len(token) >= 2 and ((token[0], token[-1]) in
                                        [('"', '"'), ("'", "'")]):
                    # It's a quoted string
                    yield 'string', token[1:-1]
                else:
                    yield 'check', self._parse_check(clean)

            # Yield the trailing parens
            for i in range(trail):
                yield ')', ')'
1201727
def tokenize_string(source):
    
    line_reader = StringIO(source).readline
    token_generator = tokenize.generate_tokens(line_reader)

    # Loop over all tokens till a backtick (`) is found.
    # Then, take all tokens till the next backtick to form a backtick quoted
    # string.
    for toknum, tokval, _, _, _ in token_generator:
        if tokval == '`':
            tokval = " ".join(it.takewhile(
                lambda tokval: tokval != '`',
                map(operator.itemgetter(1), token_generator)))
            toknum = _BACKTICK_QUOTED_STRING
        yield toknum, tokval
865535
def _get_error_context(input_, token):
    

    try:
        line = input_[token.lexpos: input_.index('\n', token.lexpos)]
    except ValueError:
        line = input_[token.lexpos:]

    i = input_.rfind('\n', 0, token.lexpos)
    if i < 0:
        i = 0
    line = input_[i:token.lexpos] + line
    lines = [line.strip('\r\n')]
    col = token.lexpos - i
    while len(lines) < 5 and i > 0:
        end = i
        i = input_.rfind('\n', 0, i)
        if i < 0:
            i = 0
        lines.insert(0, input_[i:end].strip('\r\n'))
    pointer = ''
    for dummy_ch in str(token.value):
        pointer += '^'
    pointline = ''
    i = 0
    while i < col - 1:
        if lines[-1][i].isspace():
            pointline += lines[-1][i]
            # otherwise, tabs complicate the alignment
        else:
            pointline += ' '
        i += 1
    lines.append(pointline + pointer)
    return lines
1329124
@Requires("nextToken == null")
  @Ensures("result == (nextToken != null)")
  protected boolean lex() throws IOException {
    int startOffset = currentOffset;
    StringBuilder buffer = new StringBuilder();
    int c = readChar(true);
    if (c == -1) {
      return false;
    }
    buffer.append((char) c);
    switch (c) {
      case '/':
        /* Comments. */
        c = readChar(true);
        switch (c) {
          case '/':
            buffer.append((char) c);
            do {
              c = readChar(false);
              buffer.append((char) c);
            } while (c != '\n');
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          case '*':
            buffer.append((char) c);
            for (;;) {
              c = readChar(false);
              buffer.append((char) c);
              if (c == '*') {
                c = readChar(false);
                buffer.append((char) c);
                if (c == '/') {
                  break;
                }
              }
            }
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          default:
            unreadChar(c);
            nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                  startOffset);
        }
        break;

      case '\'':
      case '"':
        /* Quoted strings. */
        int delim = c;
        for (;;) {
          c = readChar(false);
          buffer.append((char) c);
          if (c == delim) {
            break;
          }
          if (c == '\\') {
            buffer.append((char) readChar(false));
          }
        }
        nextToken = new Token(TokenKind.QUOTE, buffer.toString(), startOffset);
        break;

      default:
        if (Character.isJavaIdentifierStart(c)) {
          /* Identifiers. */
          while ((c = readChar(true)) != -1
                 && Character.isJavaIdentifierPart(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.WORD, buffer.toString(), startOffset);
        } else if (Character.isWhitespace(c)) {
          /* White space. */
          while ((c = readChar(true)) != -1 && Character.isWhitespace(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.SPACE, buffer.toString(),
                                startOffset);
        } else {
          /* Symbol. */
          nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                startOffset);
        }
    }
    return true;
  }