Consumes the next token from the token stream.
@return {int} The token type of the token that was just consumed.
@method get
328232
function(channel){
    
        var tokenInfo   = this._tokenData,
            reader      = this._reader,
            value,
            i           =0,
            len         = tokenInfo.length,
            found       = false,
            token,
            info;
            
        //check the lookahead buffer first
        if (this._lt.length && this._ltIndex >= 0 && this._ltIndex < this._lt.length){  
                           
            i++;
            this._token = this._lt[this._ltIndex++];
            info = tokenInfo[this._token.type];
            
            //obey channels logic
            while((info.channel !== undefined && channel !== info.channel) &&
                    this._ltIndex < this._lt.length){
                this._token = this._lt[this._ltIndex++];
                info = tokenInfo[this._token.type];
                i++;
            }
            
            //here be dragons
            if ((info.channel === undefined || channel === info.channel) &&
                    this._ltIndex <= this._lt.length){
                this._ltIndexCache.push(i);
                return this._token.type;
            }
        }
        
        //call token retriever method
        token = this._getToken();

        //if it should be hidden, don't save a token
        if (token.type > -1 && !tokenInfo[token.type].hide){
                     
            //apply token channel
            token.channel = tokenInfo[token.type].channel;
         
            //save for later
            this._token = token;
            this._lt.push(token);

            //save space that will be moved (must be done before array is truncated)
            this._ltIndexCache.push(this._lt.length - this._ltIndex + i);  
        
            //keep the buffer under 5 items
            if (this._lt.length > 5){
                this._lt.shift();                
            }
            
            //also keep the shift buffer under 5 items
            if (this._ltIndexCache.length > 5){
                this._ltIndexCache.shift();
            }
                
            //update lookahead index
            this._ltIndex = this._lt.length;
        }
            
        /*
         * Skip to the next token if:
         * 1. The token type is marked as hidden.
         * 2. The token type has a channel specified and it isn't the current channel.
         */
        info = tokenInfo[token.type];
        if (info && 
                (info.hide || 
                (info.channel !== undefined && channel !== info.channel))){
            return this.get(channel);
        } else {
            //return just the type
            return token.type;
        }
    }
554991
def get_token(self):
        ""
        if self.pushback:
            tok = self.pushback.popleft()
            return tok
        # No pushback.  Get a token.
        raw = self.read_token()
        # Handle inclusions
        if self.source is not None:
            while raw == self.source:
                spec = self.sourcehook(self.read_token())
                if spec:
                    (newfile, newstream) = spec
                    self.push_source(newstream, newfile)
                raw = self.get_token()
        # Maybe we got EOF instead?
        while raw == self.eof:
            if not self.filestack:
                return self.eof
            else:
                self.pop_source()
                raw = self.get_token()
        # Neither inclusion nor EOF
        return raw
263606
function Parser( tokenStream ){
   
    var debug = false;

    var currentToken, lastComment, types = [];
   
    function isAnyOf( type, possibleTypes ){
        for(var i = 0; i < possibleTypes.length; i++) {
            if ( possibleTypes[i] === type ){ 
                return true;
            }
        }
    
        return false;
    }

    function error( message ){
        throw new Error( message + 
            "\nAt line: " + currentToken.line + ", col: " +  currentToken.col + "\n\n" +
            tokenStream.neighborhood()
        );
    }

    function consume( expectedType ){
        currentToken = tokenStream.consume();

        if (debug) console.log("currentToken", currentToken);

        if ( expectedType && !isAnyOf( currentToken.type, Array.prototype.slice.call(arguments) )){
            error( "syntax error - expected \"" + Array.prototype.join.call( arguments, ", ") + "\", but got \"" + currentToken.type + "\"" ); 
        } 

        return currentToken;
    }

    function peak(){
        return tokenStream.peak();
    }

    function parseExtendsList(){
        
        consume("extends");

        return parseIdList();
    }
   

    function parseExtends(){
        
        consume("extends");

        // get the type name
        var r = consume();
        return r.contents;
    }
   
    function parseIdList(){

        var idlist = []
            , peaked;

        do {
            consume();

            if ( currentToken.type === "id" ){
                idlist.push( currentToken.contents );
            }
        } while ( (peaked = peak()) && (peaked.type === "id" || peaked.type === ",") )

        return idlist;
    }

    function parseImplements(){
         
        consume("implements");
        return parseIdList();

    }

    function parseClass( visibility ){
     
        // get visibility from current token or default
        visibility = visibility || "public";
        
        consume("class");  
       
        // get the type name for the class
        var typeName = consume().contents;
    
        // build the class type
        var def = new Class( typeName, visibility ); 
        def.line = currentToken.line;
    
        // set the description
        def.description = getLastComment();

        var peaked = peak(); 
        var parsedExtends = false;
    
        // parent class
        if ( peaked.type === "extends" ){
            parsedExtends = true;
            def.parentClass = parseExtends();
            peaked = peak();
        }
        
        // parse interfaces
        if ( peaked.type === "implements" ){
            def.interfaces = parseImplements();                       
            peaked = peak();
        }
 
        // parent class
        if ( peaked.type === "extends" ){
            if (parsedExtends) error("Duplicate extends encountered");
            def.parentClass = parseExtends();
        }       

        parseClassBody( def );

        if (debug) console.log(def);

        return def; 
    }

    function consumeBlock(){

        consume("{");
        var parenCount = 1;

        while ( parenCount != 0 ){
            consume();

            if (currentToken.type === "{") {
                parenCount++;
            } else if (currentToken.type === "}"){
                parenCount--;
            }
        }
    }

    function consumeWhile(){
        
        var possibleTypes = Array.prototype.slice.call(arguments);
        
        var consumed = [];
        while (isAnyOf( peak().type, possibleTypes)){
            consumed.push(consume.apply(null, arguments));
        }
        return consumed;
    }

    function consumeIf(){

        var possibleTypes = Array.prototype.slice.call(arguments);
        var peaked = peak();
        if (isAnyOf(peaked.type, possibleTypes)) consume();
    }

    function consumeUntil(){

        var possibleTypes = Array.prototype.slice.call(arguments);
        var peaked;
        while ( (peaked = peak()) && !isAnyOf( peaked.type, possibleTypes) ){
            consume();
        }
    }

    function consumeUntilInclusive(char){
        consumeUntil(char);
        consume(char);
    }

    function parseProperty( isStatic ){

        var currentLine = currentToken.line;

        consume("var");

        var name = consume().contents;
        var type;

        var peaked = peak();
        
        if (peaked.type === ":"){
           
            consume(":");
            type = consume("id").contents; 
            peaked = peak();
            
        }
       
        consumeUntilInclusive(";");

        var prop = new Property( name, type, getLastComment() ); 
        prop.line = currentLine;
        return prop;
    }
  
    function parseMethodArgument(){
    
        // id : Type = expression
        var name = consume("id").contents;
      
        var peaked = peak();

        var type;
        if (peaked.type === ":"){
            type = parseTypeAnnotation();
            peaked = peak();
        }

        var defaultVal;
        if (peaked.type === "="){    
            consume("=");
            defaultVal = consume("number", "string", "boolean", "null").contents; 
        }

        return new MethodArgument(name, type, defaultVal);
    }

    function parseTypeAnnotation(){
        consume(":");
        var ids = consumeWhile("id","->");
        return ids.reduce((a,x) => x.contents + a, "");
    }

    function parseMethodArguments(){

        // ( methodArgument, methodArgument )
        consume("(");

        var peaked, args = [];

        while ( (peaked = peak()) && peaked.type != ")"){
            
            if (peaked.type === "id"){
                args.push( parseMethodArgument() );
            } else if (peaked.type === ","){
                consume(","); // consume the ","
            } else {
                error("Failed to parse method arguments!");                
            }

        }
       
        consume(")");
        return args;
    }

    function getLastComment(){

        var lc = lastComment;
        lastComment = undefined;
        return lc ? lc.contents : lc;

    }

    function parseMethod( isStatic ){
    
        var currentLine = currentToken.line;

        // function MethodName( methodArgument, methodArgument ) : ExpectedType { ... } 
        consume("function");

        var name = consume("id", "new").contents;
        
        var args = parseMethodArguments();

        peaked = peak();
   
        var type;
        if (peaked.type === ":"){
           type = parseTypeAnnotation(); 
        }
    
        consumeBlock();

        var method = new Method( name, args, type, getLastComment() );
        method.line = currentLine;
        method.isStatic = isStatic;
        return method;
    }

    function parseClassMember( def, visibility ){

        // parse "public" or "private"
        var visibility = visibility ? visibility : consume("visibility").contents;     

        // hack to ignore private members
        if ( visibility === "private"){
            def = new Class();
        }

        var peaked = peak();
        var isStatic = false;
    
        if ( peaked.type === "static" ){
            isStatic = true;
            
            consume("static");
            peaked = peak();
        }

        if ( peaked.type === "inline" ){
           
            // TODO - parse inline
            consume("inline");
            peaked = peak();
        }
        
        if ( peaked.type === "var" ){
            return def.properties.push( parseProperty( isStatic ));
        } else if (peaked.type === "function"){
            return def.methods.push( parseMethod( isStatic )); 
        }

        throw new Error("Unknown class member encountered" + JSON.stringify( peaked ));
    }

    function parseClassBody( def ){

        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "visibility"){
                parseClassMember( def );
                continue;
            } else if (peaked.type === "function"){
                parseClassMember( def, "private" );
                continue;
            } else if (peaked.type === "var"){
                parseClassMember( def, "private" );
                continue;
            } else if (peaked.type === "static"){
                parseClassMember( def, "private" );
                continue; 
            }

            consume();
        }

        consume("}");
    }

    function parseMethodDefinition(){
    
        // function MethodName( methodArgument, methodArgument ) : ExpectedType; 
        consume("function");

        var currentLine = currentToken.line;
        var name = consume("id", "new").contents;
        
        var args = parseMethodArguments();

        peaked = peak();
   
        var type;
        if (peaked.type === ":"){
           type = parseTypeAnnotation(); 
        }
    
        consumeUntilInclusive(";");

        var method = new Method( name, args, type, getLastComment() );
        method.line = currentLine;
        return method;
    }

    function parseInterfaceMember( def, visibility ){

        var visibility = visibility ? visibility : consume("visibility").contents;     

        // hack to ignore private members
        if ( visibility === "private"){
            def = new Interface();
        }

        var peaked = peak();
    
        if ( peaked.type === "var" ){
            return def.properties.push( parseProperty());
        } else if (peaked.type === "function"){
            return def.methods.push( parseMethodDefinition()); 
        }

        throw new Error("Unknown interface member type encountered" + JSON.stringify( peaked ));
    }
    
    function parseInterfaceBody( def ){

        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "visibility"){
                parseInterfaceMember( def );
                continue;
            } else if (peaked.type === "function"){
                parseInterfaceMember( def, "public" );
                continue;
            }  

            consume();
        }

        consume("}");
    }

    function parseInterface( visibility ){

        visibility = visibility || "public";
        
        consume("interface");  
       
        // get the type name for the class
        var typeName = consume().contents;
    
        var def = new Interface( typeName, visibility ); 
        def.line = currentToken.line; 
        def.description = getLastComment();
        
        if ( peak().type === "extends" ){
            def.interfaces = parseExtendsList();
        }
        
        parseInterfaceBody( def );

        if (debug) console.log( def );

        return def; 
    }

    function parseTypedefBody( def ){

        def.isAnonymousObject = true;
        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "var"){
                def.anonymousObject.properties.push(parseProperty());
                continue;
            } 

            consume();
        }

        consume("}");
    }

    function parseTypedef( visibility ){
      
        visibility = visibility || "public";
        
        consume("typedef");  
       
        var typeName = consume().contents;
   
        consume("=");

        var def = new Typedef( typeName, visibility ); 
        def.line = currentToken.line;
        def.description = getLastComment();
      
        // anonymous object body
        if (peak().type === "{"){
            parseTypedefBody( def );
        } else {
           def.alias = consume("id").contents;
        }

        if( debug ) console.log( def );

        return def; 
    }

    function parseComment() {
        var squashed = consume("comment");

        var peaked;
        while ( (peaked = peak()) && peaked.type === "comment" ){
            currentToken = consume();
            
            squashed.contents += "\n";
            squashed.contents += currentToken.contents; 
        }
    
        return lastComment = squashed;
    }

    function parseType(){ 
        
        var peaked = peak();
        var visibility = "public";
        
        if (peaked.type === "visibility"){
            consume("visibility");
            peaked = peak();
        }

        if (peaked.type === "class"){
            return parseClass( visibility ); 
        } else if (peaked.type === "interface"){
            return parseInterface( visibility ); 
        } else if (peaked.type === "typedef"){
            return parseTypedef( visibility );    
        } 

        error("Parser for type not implemented");    
    }
    
    this.parse = () => {
       
        var peaked;

        var types = [];
    
        while ( (peaked = peak()) ){
            if (peaked.type === "expose"){
                consume("expose"); 
                types.push( parseType() ); 
            } else if (peaked.type === "class"){
                parseClass(); 
            } else if (peaked.type === "typedef"){
                types.push( parseTypedef() ); 
            } else if (peaked.type === "interface"){
                types.push( parseInterface() ); 
            } else if ( peaked.type === "visibility" ){
                if (peaked.contents === "private") {
                    // parse, but don't add to types
                    parseType();
                    continue;
                }
                types.push( parseType() ); 
            } else if (peaked.type === "comment"){
                parseComment();
            } else {
                consume();
            }
        }

        return types;
    }
}
169246
function() {
    var captures;
    if (captures = /^([.]{1,3}|&&|\|\||[!<>=?:]=|\*\*|[-+*\/%]=?|[,=?:!~<>&\[\]])([ \t]*)/.exec(this.str)) {
      var op = captures[1];
      this.skip(captures);
      op = alias[op] || op;
      var tok = new Token(op, op);
      tok.space = captures[2];
      this.isURL = false;
      return tok;
    }
  }
398055
public void consumeUntil(Parser recognizer, IntervalSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) {
        CSSToken t;
        boolean finish;
        TokenStream input = recognizer.getInputStream();
        do {
            Token next = input.LT(1);
            if (next instanceof CSSToken) {
                t = (CSSToken) input.LT(1);
                if (t.getType() == Token.EOF) {
                    logger.trace("token eof ");
                    break;
                }
            } else
                break; /* not a CSSToken, probably EOF */
            // consume token if does not match
            finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.contains(t.getType()));
            if (!finish) {
                logger.trace("Skipped: {}", t);
                input.consume();
            }
        } while (!finish);
    }
1198592
public Token lookahead() {
        Token current = token;
        if (current.next == null) {
            current.next = token_source.getNextToken();
        }
        return current.next;
    }
622457
def consume_token(self, tokens, index, tokens_len):
        
        if _is_really_comment(tokens, index):
            self.last_line_with_comment = tokens[index].line

        finished = False

        if (not _is_in_comment_type(tokens[index].type) and
                self.last_line_with_comment != tokens[index].line):
            finished = True
            end = index
        elif index == (tokens_len - 1):
            finished = True
            end = index + 1

        if finished:
            return _paste_tokens_line_by_line(tokens,
                                              TokenType.RST,
                                              self.begin,
                                              end)
669324
function consume() {
            var consumer = arguments.length <= 0 || arguments[0] === undefined ? function () {} : arguments[0];

            return Channel.consume(this, consumer);
        }
514199
def next(self):
        """"""
        rv = self.current
        if self._pushed:
            self.current = self._pushed.popleft()
        elif self.current.type is not TOKEN_EOF:
            try:
                self.current = self._next()
            except StopIteration:
                self.close()
        return rv
1163510
@Nullable
    public Token next() throws IOException {
        if (unreadToken != null) {
            lastToken = unreadToken;
            unreadToken = null;
            return lastToken;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == Token.kNewLine ||
                lastChar == Token.kCarriageReturn ||
                lastChar == Token.kSpace ||
                lastChar == Token.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == Token.kShellComment) {
                getRestOfLine();
                continue;
            }

            if (lastChar == Token.kLiteralDoubleQuote || lastChar == Token.kLiteralQuote) {
                lastToken = nextStringLiteral((char) lastChar);
                return lastToken;
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                lastToken = nextNumber();
                return lastToken;
            } else if ('_' == lastChar ||
                       ('a' <= lastChar && lastChar <= 'z') ||
                       ('A' <= lastChar && lastChar <= 'Z')) {
                lastToken = nextToken();
                return lastToken;
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // non-ASCII UTF-8 characters are only allowed inside JSON string literals.
                throw failure(lineNo, linePos, 1, "Unknown token initiator '%s'", Strings.escape((char) lastChar));
            } else {
                lastToken = nextSymbol();
                return lastToken;
            }
        }

        return null;
    }
69555
public void nextToken() {
        byte ct[] = ctype;
        int c;
        int lctype;
        sval = null;
        isQuoted = false;

        do {
            c = read();
            if (c < 0) {
                return;
            }
            lctype = (c < 256) ? ct[c] : unicode2ctype(c);
        } while (lctype == CT_WHITESPACE);

        if (lctype == CT_ALPHA) {
            int i = 0;
            do {
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char) c;
                c = read();
                lctype = c < 0 ? CT_WHITESPACE : (c < 256)? ct[c] : unicode2ctype(c);
            } while (lctype == CT_ALPHA);
            if (c >= 0) --next; // push last back
            sval = String.copyValueOf(buf, 0, i);
            return;
        }

        if (lctype == CT_QUOTE) {
            int quote = c;
            int i = 0;
            /* Invariants (because \Octal needs a lookahead):
             *   (i)  c contains char value
             *   (ii) d contains the lookahead
             */
            int d = read();
            while (d >= 0 && d != quote) {
                if (d == '\\') {
                    c = read();
                    int first = c;   /* To allow \377, but not \477 */
                    if (c >= '0' && c <= '7') {
                        c = c - '0';
                        int c2 = read();
                        if ('0' <= c2 && c2 <= '7') {
                            c = (c << 3) + (c2 - '0');
                            c2 = read();
                            if ('0' <= c2 && c2 <= '7' && first <= '3') {
                                c = (c << 3) + (c2 - '0');
                                d = read();
                            } else
                                d = c2;
                        } else
                          d = c2;
                    } else {
                        switch (c) {
                        case 'a':
                            c = 0x7;
                            break;
                        case 'b':
                            c = '\b';
                            break;
                        case 'f':
                            c = 0xC;
                            break;
                        case 'n':
                            c = '\n';
                            break;
                        case 'r':
                            c = '\r';
                            break;
                        case 't':
                            c = '\t';
                            break;
                        case 'v':
                            c = 0xB;
                            break;
                        }
                        d = read();
                    }
                } else {
                    c = d;
                    d = read();
                }
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char)c;
            }

            if (d == quote) {
                isQuoted = true;
            }
            sval = String.copyValueOf(buf, 0, i);
        }
    }
903232
public function consume()
	{
		if(!$this->valid())
		{
			throw new \OutOfBoundsException('No more serialized token data available');
		}

		$token = $this->current();
		$this->next();

		return $token;
	}
653501
function() {
      var result = new StringBuffer();
      var array = this._input.split('');
      // var prev = null;

      for (var i = 0; i < array.length; i++) {
        var cur = array[i];
        var next = array[i + 1];

        if (typeof next !== 'undefined') {
          var curToken = cur + next;

          if (this._mode.tokens.ia[curToken]) {
            var nextNext = array[i + 2];
            if (typeof nextNext === 'undefined' || /^[-\s]$/.test(nextNext)) {
              result.append(this._mode.tokens.ia[curToken]);
              i++;
              continue;
            }
          }
        }

        if (this._mode.cyr2lat[cur]) {
          result.append(this._mode.cyr2lat[cur]);
        } else {
          result.append(cur);
        }

        // prev = cur;
      }

      return result.toString();
    }
246778
function (tokens) {
        var token = null;
        for (var i = 0; i < tokens.length; i++) {
            // Find the most specific token.
            if (tokenFromAceToken[tokens[i]] !== undefined) {
                token = tokenFromAceToken[tokens[i]];
            }
        }
        return token;
    }
1273145
function consumeIdent(stream) {
	stream.start = stream.pos;
	stream.eatWhile(isIdentPrefix);
	stream.eatWhile(isIdent);
	return stream.start !== stream.pos ? stream.current() : null;
}
531574
public String nextToken() 
    {
        if (m_tokenOffset_ < 0) {
            if (m_nextOffset_ < 0) {
                throw new NoSuchElementException("No more tokens in String");   
            }
            // pre-calculations of tokens not done
            if (m_returnDelimiters_) {
                int tokenlimit = 0;
                int c = UTF16.charAt(m_source_, m_nextOffset_);
                boolean contains = delims == null 
                    ? m_delimiters_.contains(c) 
                    : c < delims.length && delims[c];
                if (contains) {
                     if (m_coalesceDelimiters_) {
                        tokenlimit = getNextNonDelimiter(m_nextOffset_);
                     } else {
                        tokenlimit = m_nextOffset_ + UTF16.getCharCount(c);
                        if (tokenlimit == m_length_) {
                            tokenlimit = -1;
                        }
                     }
                }
                else {
                    tokenlimit = getNextDelimiter(m_nextOffset_);
                }
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                }
                m_nextOffset_ = tokenlimit;
                return result;
            }
            else {
                int tokenlimit = getNextDelimiter(m_nextOffset_);
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                    m_nextOffset_ = tokenlimit;
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                    m_nextOffset_ = getNextNonDelimiter(tokenlimit);
                }
                
                return result;
            }
        }
        // count was called before and we have all the tokens
        if (m_tokenOffset_ >= m_tokenSize_) {
            throw new NoSuchElementException("No more tokens in String");
        }
        String result;
        if (m_tokenLimit_[m_tokenOffset_] >= 0) {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_],
                                         m_tokenLimit_[m_tokenOffset_]);
        }
        else {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_]);
        }
        m_tokenOffset_ ++;
        m_nextOffset_ = -1;
        if (m_tokenOffset_ < m_tokenSize_) {
            m_nextOffset_ = m_tokenStart_[m_tokenOffset_];
        }
        return result;
    }
582175
function(token) {
  if (token === undefined || token === null) {
    return {
      nextPartitionKey: undefined,
      nextRowKey:       undefined,
    };
  }
  assert(typeof token === 'string', 'Continuation token must be a string if ' +
                                     'not undefined');
  // Split at tilde (~)
  token = token.split('~');
  assert(token.length === 2, 'Expected an encoded continuation token with ' +
                             'a single tilde as separator');
  return {
    nextPartitionKey: decodeURIComponent(token[0]),
    nextRowKey:       decodeURIComponent(token[1]),
  };
}