Get a token from the input stream (or from stack if it's nonempty)
554991
def get_token(self):
        ""
        if self.pushback:
            tok = self.pushback.popleft()
            return tok
        # No pushback.  Get a token.
        raw = self.read_token()
        # Handle inclusions
        if self.source is not None:
            while raw == self.source:
                spec = self.sourcehook(self.read_token())
                if spec:
                    (newfile, newstream) = spec
                    self.push_source(newstream, newfile)
                raw = self.get_token()
        # Maybe we got EOF instead?
        while raw == self.eof:
            if not self.filestack:
                return self.eof
            else:
                self.pop_source()
                raw = self.get_token()
        # Neither inclusion nor EOF
        return raw
1042611
def _handle_token(self, token):
        """"""
        try:
            return _HANDLERS[type(token)](self, token)
        except KeyError:
            err = "_handle_token() got unexpected {0}"
            raise ParserError(err.format(type(token).__name__))
387606
@SuppressWarnings("squid:S1067")
    protected Token fetchSymbol() {
        Token result = Token.create(Token.TokenType.SYMBOL, input.current());
        result.addToTrigger(input.consume());
        if (result.isSymbol("*") && input.current().is('*')
            || result.isSymbol("&") && input.current().is('&')
            || result.isSymbol("|") && input.current().is('|')
            || result.isSymbol() && input.current().is('=')) {
            result.addToTrigger(input.consume());
        }
        return result;
    }
955470
def expr(self, token_stream, token):
        
        lval = self.keyword(token_stream, token, {})
        token = next(token_stream)
        if token[0] == 'OR':
            op = or_
            token = next(token_stream)
        elif token[0] == 'TEXT' or token[0] == 'COLOR':
            op = lambda l, r: list(flatten([l, r]))
        elif token[0] == 'SEPARATOR':
            op = lambda l, r: list(flatten([l, r]))
            token = next(token_stream)
        else:
            return [lval]
        rval = self.expr(token_stream, token)
        return op(lval, rval)
277097
private String getSvalIgnoringBom(StreamTokenizer tokeniser, Reader in, String token) {
        if(tokeniser.sval != null) {
            if(tokeniser.sval.contains(token)) {
                return token;
            }
        }
        return tokeniser.sval;
    }
914384
def _token_of(self, input):
        

        if isinstance(input, dict):

            # Intrinsic functions are always dicts
            if is_intrinsics(input):
                # Intrinsic functions are handled *exactly* like a primitive type because
                # they resolve to a primitive type when creating a stack with CloudFormation
                return self.TOKEN.PRIMITIVE
            else:
                return self.TOKEN.DICT

        elif isinstance(input, list):
            return self.TOKEN.LIST

        else:
            return self.TOKEN.PRIMITIVE
180178
def _peek(tokens, n=0):
    """"""
    return tokens.peek(n=n, skip=_is_comment, drop=True)
895742
function TokenStreamBase(input, tokenData){

    /**
     * The string reader for easy access to the text.
     * @type StringReader
     * @property _reader
     * @private
     */
    this._reader = input ? new StringReader(input.toString()) : null;

    /**
     * Token object for the last consumed token.
     * @type Token
     * @property _token
     * @private
     */
    this._token = null;

    /**
     * The array of token information.
     * @type Array
     * @property _tokenData
     * @private
     */
    this._tokenData = tokenData;

    /**
     * Lookahead token buffer.
     * @type Array
     * @property _lt
     * @private
     */
    this._lt = [];

    /**
     * Lookahead token buffer index.
     * @type int
     * @property _ltIndex
     * @private
     */
    this._ltIndex = 0;

    this._ltIndexCache = [];
}
576118
def NextToken(self):
    """"""
    # Nothing in the input stream - no token can match.
    if not self.buffer:
      return

    current_state = self.state
    for token in self._tokens:
      # Does the rule apply to us?
      if token.state_regex and not token.state_regex.match(current_state):
        continue

      if self.verbose:
        logging.debug("%s: Trying to match %r with %r", self.state,
                      self.buffer[:10], token.re_str)

      # Try to match the rule
      m = token.regex.match(self.buffer)
      if not m:
        continue

      if self.verbose:
        logging.debug("%s matched %s", token.re_str, m.group(0))

      # A token matched the empty string. We can not consume the token from the
      # input stream.
      if m.end() == 0:
        raise RuntimeError("Lexer bug! Token can not match the empty string.")

      # The match consumes the data off the buffer (the handler can put it back
      # if it likes)
      self.processed_buffer += self.buffer[:m.end()]
      self.buffer = self.buffer[m.end():]
      self.processed += m.end()

      next_state = token.next_state
      for action in token.actions:
        if self.verbose:
          logging.debug("Calling %s with %s", action, m.group(0))

        # Is there a callback to handle this action?
        cb = getattr(self, action, self.Default)

        # Allow a callback to skip other callbacks.
        try:
          possible_next_state = cb(string=m.group(0), match=m)
          if possible_next_state == "CONTINUE":
            continue
          # Override the state from the Token
          elif possible_next_state:
            next_state = possible_next_state
        except ParseError as e:
          self.Error(e)

      # Update the next state
      if next_state:
        self.state = next_state

      return token

    # Check that we are making progress - if we are too full, we assume we are
    # stuck.
    self.Error("Lexer stuck at state %s" % (self.state))
    self.processed_buffer += self.buffer[:1]
    self.buffer = self.buffer[1:]
    return "Error"
932876
function(context: ParserContext): ?Token {
    if (
      !util.IsFollowsRuleSatisfied(
        context.getTokenWalker(),
        [
          { token: "keyword", values: ["goto"] },
          { token: "operator", values: [":"], optional: true }
        ],
        true
      )
    )
      return null;

    let value = context.reader.read();
    while (!context.reader.isEOF() && !/[\W]/.test(context.reader.peek()))
      value += context.reader.read();

    return context.createToken("label", value);
  }
572574
private static void processToken( String token, Stack<String> stack, Properties props )
    {
        if( "}".equals( token ) )
        {
            String name = stack.pop();
            String open = stack.pop();
            if( "${".equals( open ) )
            {
                startProperty( name, props, stack );
            }
            else
            {
                push( stack, "${" + name + "}" );
            }
        }
        else
        {
            if( "$".equals( token ) )
            {
                stack.push( "$" );
            }
            else
            {
                push( stack, token );
            }
        }
    }
952021
def tokenize(self, source, name=None, filename=None, state=None):
        
        stream = self.tokeniter(source, name, filename, state)
        return TokenStream(self.wrap(stream, name, filename), name, filename)
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }
667613
def parse(input_: Union[str, FileStream], source: str) -> Optional[str]:
    

    # Step 1: Tokenize the input stream
    error_listener = ParseErrorListener()
    if not isinstance(input_, FileStream):
        input_ = InputStream(input_)
    lexer = jsgLexer(input_)
    lexer.addErrorListener(error_listener)
    tokens = CommonTokenStream(lexer)
    tokens.fill()
    if error_listener.n_errors:
        return None

    # Step 2: Generate the parse tree
    parser = jsgParser(tokens)
    parser.addErrorListener(error_listener)
    parse_tree = parser.doc()
    if error_listener.n_errors:
        return None

    # Step 3: Transform the results the results
    parser = JSGDocParser()
    parser.visit(parse_tree)

    if parser.undefined_tokens():
        for tkn in parser.undefined_tokens():
            print("Undefined token: " + tkn)
        return None

    return parser.as_python(source)
348453
def _get_token(self):
        """"""
        # HTTP request
        try:
            raw_res = yield from self._session.get(TOKEN_URL,
                                                   headers=self._headers,
                                                   timeout=self._timeout)
        except OSError:
            raise PyFidoError("Can not get token")
        # Research for json in answer
        content = yield from raw_res.text()
        reg_res = re.search(r"\({.*}\)", content)
        if reg_res is None:
            raise PyFidoError("Can not finf token json")
        # Load data as json
        return_data = json.loads(reg_res.group()[1:-1])
        # Get token and uuid
        token = return_data.get('result', {}).get('accessToken')
        uuid = return_data.get('result', {}).get('userData', {}).get('uuid')
        # Check values
        if token is None or uuid is None:
            raise PyFidoError("Can not get token or uuid")

        return token, uuid
225248
function token(input) {
  let token;
  
  if (typeof input === "string") {
    token = input;
  } else if (Buffer.isBuffer(input)) {
    token = input.toString("hex");
  }

  token = token.replace(/[^0-9a-f]/gi, "");

  if (token.length === 0) {
    throw new Error("Token has invalid length");
  }

  return token;
}
324233
private static Optional<Token<?>> getTokenFromSeqFile(String authPath, String proxyUserName) throws IOException {
    try (Closer closer = Closer.create()) {
      FileSystem localFs = FileSystem.getLocal(new Configuration());
      SequenceFile.Reader tokenReader =
          closer.register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf()));
      Text key = new Text();
      Token<?> value = new Token<>();
      while (tokenReader.next(key, value)) {
        LOG.info("Found token for " + key);
        if (key.toString().equals(proxyUserName)) {
          return Optional.<Token<?>> of(value);
        }
      }
    }
    return Optional.absent();
  }