Collect the tokens from a (potentially) nested parse result.
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens
396923
function extract(token) {
        var obj = {};
        Object.keys(token).map(function(k) {
            if ( !ignore.has(k) ) {
                obj[k] = token[k];
            }
        });
        return obj;
    }
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
1364673
def parse_tokens(self, tokens):
        
        if len(tokens) == 1:
            return list(), tokens, {"kubectl": self.ast.help}
        else:
            tokens.reverse()
        parsed, unparsed, suggestions = self.treewalk(self.ast, parsed=list(), unparsed=tokens)
        if not suggestions and unparsed:
            # TODO: @vogxn: This is hack until we include expected value types for each option and argument.
            # Whenver we recieve no suggestions but are left with unparsed tokens we pop the value and walk the
            # tree again without values
            logger.debug("unparsed tokens remain, possible value encountered")
            unparsed.pop()
            parsed.reverse()
            unparsed.extend(parsed)
            logger.debug("resuming treewalk with tokens: %s", unparsed)
            return self.treewalk(self.ast, parsed=list(), unparsed=unparsed)
        else:
            return parsed, unparsed, suggestions
1320730
def make_tokens(parse_buffer):
    
    tokens = []
    for token_type, result in parse_buffer:
        token = token_type(result)
        if token is not None:
            tokens.append(token)
    return tokens
653220
def parse_stream_raw(self, stream, debug=False):
        """"""
        tokens = tokenize.generate_tokens(stream.readline)
        return self.parse_tokens(tokens, debug)
576908
private Map<String, ParseTreeNode> getTokens() throws GrammarException,
	    TreeException {
	Map<String, ParseTreeNode> tokens = new HashMap<String, ParseTreeNode>();
	ParseTreeNode tokensTree = parserTree.getChild("Tokens");
	ParseTreeNode tokenDefinitions = tokensTree.getChild("TokenDefinitions");
	for (ParseTreeNode tokenDefinition : tokenDefinitions
		.getChildren("TokenDefinition")) {
	    // identifier...
	    String identifier = tokenDefinition.getChild("IDENTIFIER")
		    .getText();
	    // token parts...
	    tokens.put(identifier, tokenDefinition);
	    // visibility...
	    ParseTreeNode visibilityAST = tokenDefinition.getChild("Visibility");
	    if (visibilityAST.hasChild("HIDE")) {
		tokenVisibility.put(identifier, Visibility.HIDDEN);
	    } else if (visibilityAST.hasChild("IGNORE")) {
		tokenVisibility.put(identifier, Visibility.IGNORED);
	    } else {
		tokenVisibility.put(identifier, Visibility.VISIBLE);
	    }
	}
	return tokens;
    }
919321
def parse_header(cls, value='*/*'):
        """"""
        yield from sorted((
            cls.from_string(token.strip())
            for token in value.split(',')
            if token.strip()
        ), reverse=True)
511048
def data(self):
        """"""
        if self._data is None:
            # reset after possible parsing failure
            self.__init__(self.tokens)
            return self._parse()
        else:
            return self._data
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
1311504
def _get(self, key, parser_result):
        
        try:
            list_data = parser_result[key].asList()
            if any(isinstance(obj, str) for obj in list_data):
                txt_lines = [''.join(list_data)]
            else:
                txt_lines = [''.join(f) for f in list_data]
        except KeyError:
            txt_lines = []
        return txt_lines
25678
def parse(scanner, rules, scope = ParserScope.empty)
      backtracking(scanner) do
        if scope = parse_children(scanner, rules, scope.nest)
          yield scope if block_given?
          parse_result(scope)
        end
      end
    end
1257674
def parse(self):
        
        #print("Parsing: %s"%self.parse_string)
        self.tokenize()
        if self.debug: print("Tokens found: %s"%self.token_list)
        
        try:
            parse_tree = self.parse2()
        except Exception as e:
                raise e
        return parse_tree
998302
def get_tokens_unprocessed(self, text, stack=('root',)):
        """"""
        self.content_type = None
        return RegexLexer.get_tokens_unprocessed(self, text, stack)
5160
def patched_nested_parse(self, *args, **kwargs):
        """"""
        kwargs["match_titles"] = True
        return self.stored_nested_parse(*args, **kwargs)
309445
def _parse_tokenize(self, rule):
        """"""

        for token in self._TOKENIZE_RE.split(rule):
            # Skip empty tokens
            if not token or token.isspace():
                continue

            # Handle leading parens on the token
            clean = token.lstrip('(')
            for i in range(len(token) - len(clean)):
                yield '(', '('

            # If it was only parentheses, continue
            if not clean:
                continue
            else:
                token = clean

            # Handle trailing parens on the token
            clean = token.rstrip(')')
            trail = len(token) - len(clean)

            # Yield the cleaned token
            lowered = clean.lower()
            if lowered in ('and', 'or', 'not'):
                # Special tokens
                yield lowered, clean
            elif clean:
                # Not a special token, but not composed solely of ')'
                if len(token) >= 2 and ((token[0], token[-1]) in
                                        [('"', '"'), ("'", "'")]):
                    # It's a quoted string
                    yield 'string', token[1:-1]
                else:
                    yield 'check', self._parse_check(clean)

            # Yield the trailing parens
            for i in range(trail):
                yield ')', ')'
962213
def get_token_and_data(self, data):
        '''
        
        '''
        token = ''
        for c in data:
            if c != ')':
                token = token + c
            else:
                break;

        return token, data.lstrip(token + '):')