Parse the token stream into a nice dictionary data structure.
557479
def _parse(self):
        """"""
        while self._cur_token['type'] in (TT.ws, TT.lbreak):
            self._skip_whitespace()
            self._skip_newlines()

        self._data = self._parse_value()

        return self._data
214068
def parse(self, stream):
        """"""

        items = OrderedDict()
        for i, line in enumerate(stream):
            line = line.strip()
            if not line or line[0] in ["#", ";", "["] or line.startswith("---"):
                continue
            white_space = "\\s*"
            key = "(?P<key>[^:=;#\s]+?)"
            value = white_space+"[:=\s]"+white_space+"(?P<value>.+?)"
            comment = white_space+"(?P<comment>\\s[;#].*)?"

            key_only_match = re.match("^" + key + comment + "$", line)
            if key_only_match:
                key = key_only_match.group("key")
                items[key] = "true"
                continue

            key_value_match = re.match("^"+key+value+comment+"$", line)
            if key_value_match:
                key = key_value_match.group("key")
                value = key_value_match.group("value")

                if value.startswith("[") and value.endswith("]"):
                    # handle special case of lists
                    value = [elem.strip() for elem in value[1:-1].split(",")]

                items[key] = value
                continue

            raise ConfigFileParserException("Unexpected line %s in %s: %s" % (i,
                getattr(stream, 'name', 'stream'), line))
        return items
294990
def get_token_stream(source: str) -> CommonTokenStream:
    
    lexer = LuaLexer(InputStream(source))
    stream = CommonTokenStream(lexer)
    return stream
1245764
def parse(self, text):
        """"""
        text = self.preprocess(text)
        token_stack = []
        last_pos = 0

        # Iterate through all matched tokens
        for match in self.regex.finditer(text):
            # Find which token has been matched by regex
            token, match_type, group = self.get_matched_token(match)

            # Get params from stack of tokens
            params = self.get_params(token_stack)

            # Should we skip interpreting tokens?
            skip = token_stack[-1].skip if token_stack else False

            # Check for end token first
            if match_type == MatchType.end:
                if not skip or token_stack[-1] == token:
                    removed = self.remove_token(token_stack, token)
                    if removed:
                        skip = False
                    else:
                        skip = True

            if not skip:
                # Append text preceding matched token
                start_pos = match.start(group)
                if start_pos > last_pos:
                    yield Segment(self.postprocess(text[last_pos:start_pos]), **params)

                # Actions specific for start token or single token
                if match_type == MatchType.start:
                    token_stack.append(token)
                elif match_type == MatchType.single:
                    single_params = params.copy()
                    single_params.update(token.params)
                    single_text = token.text if token.text is not None else match.group(group)
                    yield Segment(single_text, token=token, match=match, **single_params)

                # Move last position pointer to the end of matched token
                last_pos = match.end(group)

        # Append anything that's left
        if last_pos < len(text):
            params = self.get_params(token_stack)
            yield Segment(self.postprocess(text[last_pos:]), **params)
564574
def parse_stream(response):
    
    stream_data = []
    stream = stdout

    for data in response:
        if data:
            try:
                data = data.decode('utf-8')
            except AttributeError as e:
                logger.exception("Unable to parse stream, Attribute Error Raised: {0}".format(e))
                stream.write(data)
                continue

            try:
                normalized_data = normalize_keys(json.loads(data))
            except ValueError:
                stream.write(data)
                continue
            except TypeError:
                stream.write(data)
                continue

            if 'progress' in normalized_data:
                stream_data.append(normalized_data)
                _display_progress(normalized_data, stream)

            elif 'error' in normalized_data:
                _display_error(normalized_data, stream)

            elif 'status' in normalized_data:
                stream_data.append(normalized_data)
                _display_status(normalized_data, stream)

            elif 'stream' in normalized_data:
                stream_data.append(normalized_data)
                _display_stream(normalized_data, stream)
            else:
                stream.write(data)

    stream.flush()
    return stream_data
373827
def tokenize_block(iterable, token_types):
    
    lines = FileWrapper(iterable)
    parse_buffer = ParseBuffer()
    line = lines.peek()
    while line is not None:
        for token_type in token_types:
            if token_type.start(line):
                result = token_type.read(lines)
                if result is not None:
                    parse_buffer.append((token_type, result))
                    break
        else:  # unmatched newlines
            next(lines)
            parse_buffer.loose = True
        line = lines.peek()
    return parse_buffer
763002
private void readSparseInstance(StreamTokenizer tokenizer, AttributeDataset data, Attribute[] attributes) throws IOException, ParseException {
        double[] x = responseIndex >= 0 ? new double[attributes.length - 1] : new double[attributes.length];
        double y = Double.NaN;
        int index = -1;
        
        // Get values for all attributes.
        do {
            getNextToken(tokenizer);
            
            // end of instance
            if (tokenizer.ttype == '}') {
                break;
            }
            
            String s = tokenizer.sval.trim();
            if (index < 0) {
                index = Integer.parseInt(s);
                if (index < 0 || index >= attributes.length) {
                    throw new ParseException("Invalid attribute index: " + index, tokenizer.lineno());
                }
                
            } else {
                
                String val = s;
                if (index != responseIndex) {
                    if (val.equals("?")) {
                        x[index] = Double.NaN;
                    } else {
                        x[index] = attributes[index].valueOf(val);
                    }
                } else {
                    if (val.equals("?")) {
                        y = Double.NaN;
                    } else {
                        y = attributes[index].valueOf(val);
                    }
                }
                
                index = -1;
            }
            
        } while (tokenizer.ttype == StreamTokenizer.TT_WORD);
        
        if (Double.isNaN(y)) data.add(x); else data.add(x, y);
    }
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
1052945
def count_tf(tokens_stream):
    
    tf = defaultdict(int)
    for tokens in tokens_stream:
        for token in tokens:
            tf[token] += 1
    return tf
867795
def parse(self, text):
        """"""
        token_stream = self.lexer.tokenize(text)
        return self.expr(token_stream, next(token_stream))
686883
def parse(response):
        """Split a a=1b=2c=3 string into a dictionary of pairs"""
        tokens = {r[0]: r[1] for r in [r.split('=') for r in response.split("&")]}

        # The odd dummy parameter is of no use to us
        if 'dummy' in tokens:
            del tokens['dummy']

        """
        If we have key names that end in digits, these indicate the result set contains multiple sets
        For example, planet0=Hoth&x=1&y=-10&planet1=Naboo&x=9&y=13 is actually data for two planets

        Elements that end in digits (like tag0, tag1 for planets) are formatted like (tag0_1, tag1_1), so we rstrip
        underscores afterwards.
        """
        if re.match('\D\d+$', tokens.keys()[0]):
            # Produce a list of dictionaries
            set_tokens = []
            for key, value in tokens:
                key = re.match('^(.+\D)(\d+)$', key)
                # If the key isn't in the format (i.e. a failsafe), skip it
                if key is not None:
                    if key.group(1) not in set_tokens:
                        set_tokens[key.group(1)] = {}

                    set_tokens[key.group(1)][key.group(0).rstrip('_')] = value
            tokens = set_tokens

        return tokens
787727
def __parse_stream(self, stream, parse_line):
        """"""

        if not stream:
            raise InvalidFormatError(cause='stream cannot be empty or None')

        nline = 0
        lines = stream.split('\n')

        for line in lines:
            nline += 1

            # Ignore blank lines and comments
            m = re.match(self.LINES_TO_IGNORE_REGEX, line, re.UNICODE)
            if m:
                continue

            m = re.match(self.VALID_LINE_REGEX, line, re.UNICODE)
            if not m:
                cause = "line %s: invalid format" % str(nline)
                raise InvalidFormatError(cause=cause)

            try:
                result = parse_line(m.group(1), m.group(2))
                yield result
            except InvalidFormatError as e:
                cause = "line %s: %s" % (str(nline), e)
                raise InvalidFormatError(cause=cause)
1172566
def parse_entry(source, loc, tokens):
    
    type_ = tokens[1].lower()
    entry_type = structures.TypeRegistry.get_type(type_)
    if entry_type is None or not issubclass(entry_type, structures.Entry):
        raise exceptions.UnsupportedEntryType(
                "%s is not a supported entry type" % type_
            )
    new_entry = entry_type()
    new_entry.name = tokens[3]
    for key, value in [t for t in tokens[4:-1] if t != ',']:
        new_entry[key] = value
    return new_entry
319795
def parse(tokens):
    ''''''
    args = []
    kwargs = {}
    last = None
    for token in tokens:
        if token.startswith('--'):
            # If this is a keyword flag, but we've already got one that we've
            # parsed, then we're going to interpret it as a bool
            if last:
                kwargs[last] = True
            # See if it is the --foo=5 style
            last, _, value = token.strip('-').partition('=')
            if value:
                kwargs[last] = value
                last = None
        elif last != None:
            kwargs[last] = token
            last = None
        else:
            args.append(token)

    # If there's a dangling last, set that bool
    if last:
        kwargs[last] = True

    return args, kwargs
509172
def __parse(self, tokenlist):
        """"""
        #empty stack
        #iterate over symbollist
        tokenlist = [x for x in tokenlist]
        if not isinstance(tokenlist, list):
            raise TypeError("Expected list, got %s" % tokenlist.__class__.__name__)
        LOG.debug("get_trees: checking list: " + str(tokenlist))
        stack = [(0, Extended_S)]
        while True:
            state = stack[-1][0]
            if len(tokenlist):#FIXME: tokenlist with one element is reported as false
                token = tokenlist[0]
            else:
                token = EndSymbol()
            newdic = self.__parsertable.insert(state, token)
            action = newdic["action"]
            if action == "Fail":
                return False
            elif action == "Accept":
                return True
            if action == "Reduce":
                reductionrule = newdic["rule"]
                #TODO extract len(right side) of the rule and insert left side
                for rsymbol in reversed(reductionrule.rightside):
                    state, symbol = stack.pop() # TODO: check
                state = stack[-1][0]
                state = self.__parsertable.goto(state,reductionrule.leftside[0])
                stack.append((state, reductionrule.leftside[0]))
            elif action == "Shift":
                stack.append((newdic['dest'], tokenlist.pop(0)))
            else:
                raise ValueError("Unknown action")
        return False
1129053
def parse_stream(stream, format=u"Jæren Sparebank"):
    
    Class = formats[format.lower()]
    return Class.csv_to_transactions(stream)
519428
def data_from_stream(self, stream):
        
        parser = self._make_representation_parser(stream, self.resource_class,
                                                  self._mapping)
        return parser.run()