Parse a text file with one url per line; comment and blank
    lines are ignored.
339442
def parse_text (url_data):
    """"""
    lineno = 0
    for line in url_data.get_content().splitlines():
        lineno += 1
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        url_data.add_url(line, line=lineno)
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
787689
def parse_strings(content="", filename=None):
    
    if filename is not None:
        content = _get_content(filename=filename)

    stringset = []
    f = content
    if f.startswith(u'\ufeff'):
        f = f.lstrip(u'\ufeff')
    #regex for finding all comments in a file
    cp = r'(?:/\*(?P<comment>(?:[^*]|(?:\*+[^*/]))*\**)\*/)'
    p = re.compile(r'(?:%s[ \t]*[\n]|[\r\n]|[\r]){0,1}(?P<line>(("(?P<key>[^"\\]*(?:\\.[^"\\]*)*)")|(?P<property>\w+))\s*=\s*"(?P<value>[^"\\]*(?:\\.[^"\\]*)*)"\s*;)'%cp, re.DOTALL|re.U)
    #c = re.compile(r'\s*/\*(.|\s)*?\*/\s*', re.U)
    c = re.compile(r'//[^\n]*\n|/\*(?:.|[\r\n])*?\*/', re.U)
    ws = re.compile(r'\s+', re.U)
    end=0
    start = 0
    for i in p.finditer(f):
        start = i.start('line')
        end_ = i.end()
        key = i.group('key')
        comment = i.group('comment') or ''
        if not key:
            key = i.group('property')
        value = i.group('value')
        while end < start:
            m = c.match(f, end, start) or ws.match(f, end, start)
            if not m or m.start() != end:
                print("Invalid syntax: %s" %\
                        f[end:start])
            end = m.end()
        end = end_
        key = _unescape_key(key)
        stringset.append({'key': key, 'value': _unescape(value), 'comment': comment})
    return stringset
1221554
def inline_parse(text, verbose=False):
    """"""
    if isinstance(text, str):
        text = text.decode('utf-8')

    parsed = parse(text, verbose=verbose)

    shift = 0
    for quantity in parsed:
        index = quantity.span[1] + shift
        to_add = u' {' + unicode(quantity) + u'}'
        text = text[0:index] + to_add + text[index:]
        shift += len(to_add)

    return text
865038
def parse(self, ascii_listing):
        
        self.parsed_lines = ParsedBASIC()
        for match in self.regex_line_no.finditer(ascii_listing):
            log.info("_" * 79)
            log.info("parse line >>>%r<<<", match.group())
            line_no = int(match.group("no"))
            line_content = match.group("content")

            self.line_data = []
            self._parse_code(line_content)
            log.info("*** line %s result: %r", line_no, self.line_data)

            self.parsed_lines[line_no] = self.line_data

        return self.parsed_lines
137194
def _read_parsed(self, lines):
        
        self.log(u"Parsing fragments from parsed text format")
        pairs = []
        for line in lines:
            pieces = line.split(gc.PARSED_TEXT_SEPARATOR)
            if len(pieces) == 2:
                identifier = pieces[0].strip()
                text = pieces[1].strip()
                if len(identifier) > 0:
                    pairs.append((identifier, [text]))
        self._create_text_fragments(pairs)
885302
def readlines(self):
        """"""

        t = self.resolved_url.get_resource().get_target()
        with open(t.fspath) as f:
            return f.readlines()
300364
def _parse(self, lines, grammar, ignore_comments=False):
        
        results = []
        for c in lines:
            if c != '' and not (ignore_comments and c[0] == '#'):
                try:
                    results.append(grammar.parseString(c))
                except pyparsing.ParseException as e:
                    raise ValueError('Invalid syntax. Verify line {} is '
                        'correct.\n{}\n\n{}'.format(e.lineno, c, e))
        return results
823966
def parse_line(line):
  
  columns = line.split()
  token = columns.pop(0)
  values = [float(column) for column in columns]
  return token, values
678964
def parse (cls, line, lineno, log, cmddict=None):
    
    start  = line.find('%')
    pos    = SeqPos(line, lineno, start + 1, len(line))
    result = None

    if start >= 0:
      result = cls(line[start:], pos)

    return result
884667
def _ParseFile(self, file_obj, line_parser):
    
    lines = [
        l.strip() for l in utils.ReadFileBytesAsUnicode(file_obj).splitlines()
    ]
    try:
      for index, line in enumerate(lines):
        if line:
          line_parser(line)
    except (IndexError, KeyError) as e:
      raise parser.ParseError("Invalid file at line %d: %s" % (index + 1, e))
568523
def parse(self, line):
        """"""
        tokens = []

        self.idx = -1   # Bogus value to ensure idx = 0 after first iteration
        self.characters = iter(line)
        self.update_chars()

        while self.char != '\n':
            # Update namelist group status
            if self.char in ('&', '$'):
                self.group_token = self.char

            if self.group_token and (
                    (self.group_token, self.char) in (('&', '/'), ('$', '$'))):
                self.group_token = False

            word = ''
            if self.char in self.whitespace:
                while self.char in self.whitespace:
                    word += self.char
                    self.update_chars()

            elif self.char in ('!', '#') or self.group_token is None:
                # Abort the iteration and build the comment token
                word = line[self.idx:-1]
                self.char = '\n'

            elif self.char in '"\'' or self.prior_delim:
                word = self.parse_string()

            elif self.char.isalpha():
                word = self.parse_name(line)

            elif self.char in ('+', '-'):
                # Lookahead to check for IEEE value
                self.characters, lookahead = itertools.tee(self.characters)
                ieee_val = ''.join(itertools.takewhile(str.isalpha, lookahead))
                if ieee_val.lower() in ('inf', 'infinity', 'nan'):
                    word = self.char + ieee_val
                    self.characters = lookahead
                    self.prior_char = ieee_val[-1]
                    self.char = next(lookahead, '\n')
                else:
                    word = self.parse_numeric()

            elif self.char.isdigit():
                word = self.parse_numeric()

            elif self.char == '.':
                self.update_chars()
                if self.char.isdigit():
                    frac = self.parse_numeric()
                    word = '.' + frac
                else:
                    word = '.'
                    while self.char.isalpha():
                        word += self.char
                        self.update_chars()
                    if self.char == '.':
                        word += self.char
                        self.update_chars()

            elif self.char in Tokenizer.punctuation:
                word = self.char
                self.update_chars()

            else:
                # This should never happen
                raise ValueError

            tokens.append(word)

        return tokens
591094
def parse_line(self, line, lineno):
        """"""
        match = self.RE_TINDERBOXPRINT.match(line) if line else None
        if match:
            line = match.group('line')

            for regexp_item in self.TINDERBOX_REGEXP_TUPLE:
                match = regexp_item['re'].match(line)
                if match:
                    artifact = match.groupdict()
                    # handle duplicate fields
                    for to_field, from_field in regexp_item['duplicates_fields'].items():
                        # if to_field not present or None copy form from_field
                        if to_field not in artifact or artifact[to_field] is None:
                            artifact[to_field] = artifact[from_field]
                    artifact.update(regexp_item['base_dict'])
                    self.artifact.append(artifact)
                    return

            # default case: consider it html content
            # try to detect title/value splitting on <br/>
            artifact = {"content_type": "raw_html", }
            if "<br/>" in line:
                title, value = line.split("<br/>", 1)
                artifact["title"] = title
                artifact["value"] = value
            # or similar long lines if they contain a url
            elif "href" in line and "title" in line:
                def parse_url_line(line_data):
                    class TpLineParser(HTMLParser):

                        def handle_starttag(self, tag, attrs):
                            d = dict(attrs)
                            artifact["url"] = d['href']
                            artifact["title"] = d['title']

                        def handle_data(self, data):
                            artifact["value"] = data

                    p = TpLineParser()
                    p.feed(line_data)
                    p.close()

                # strip ^M returns on windows lines otherwise
                # handle_data will yield no data 'value'
                parse_url_line(line.replace('\r', ''))
            else:
                artifact["value"] = line
            self.artifact.append(artifact)
980266
public void parse(InputStream is) throws IOException {
		
		BufferedReader br = new BufferedReader(new InputStreamReader(
				(InputStream) is,ByteOp.UTF8));
        String read;
        boolean allowRuleFound = false;
        // true if curr or last line read was a User-agent line 
        boolean currLineUA = false; 
        boolean lastLineUA = false;
        ArrayList<String> current = null;
        while (br != null) {
            lastLineUA = currLineUA;
            do {
                read = br.readLine();
                // Skip comments & blanks
            } while ((read != null) && ((read = read.trim()).startsWith("#") ||
                read.length() == 0));
            if (read == null) {
            	br.close();
            	br = null;
            } else {
                currLineUA = false;
                int commentIndex = read.indexOf("#");
                if (commentIndex > -1) {
                    // Strip trailing comment
                    read = read.substring(0, commentIndex);
                }
                read = read.trim();
                Matcher uaMatcher = USER_AGENT_PATTERN.matcher(read);
                if (uaMatcher.matches()) {
                    String ua = uaMatcher.group(1).trim().toLowerCase();
                    if (current == null || current.size() != 0 || allowRuleFound || !lastLineUA) {
                        // only create new rules-list if necessary
                        // otherwise share with previous user-agent
                        current = new ArrayList<String>();
                    }
                    rules.put(ua, current);
                    allowRuleFound = false;
                    currLineUA = true;
                    LOGGER.fine("Found User-agent(" + ua + ") rules...");
                    continue;
                }
                Matcher disallowMatcher = DISALLOW_PATTERN.matcher(read);
                if (disallowMatcher.matches()) {
                	if (current == null) {
                        // buggy robots.txt
                    	bSyntaxErrors = true;
                        continue;
                    }
                    String path = disallowMatcher.group(1).trim();
					// Disallow: without path is just ignored.
					if (!path.isEmpty())
                    current.add(path);
                    continue;
                }
                Matcher allowMatcher = ALLOW_PATTERN.matcher(read);
                if (allowMatcher.matches()) {
                	// Mark that there was an allow rule to clear the current list for next user-agent
                	allowRuleFound = true;
                }
                // unknown line; do nothing for now
                
                // TODO: check for "Allow" lines, and flag a syntax error if 
                //       we encounter any unknown lines?
            }
        }
    }
48073
def is_file_url(url):
    """"""
    from .misc import to_text

    if not url:
        return False
    if not isinstance(url, six.string_types):
        try:
            url = getattr(url, "url")
        except AttributeError:
            raise ValueError("Cannot parse url from unknown type: {0!r}".format(url))
    url = to_text(url, encoding="utf-8")
    return urllib_parse.urlparse(url.lower()).scheme == "file"
634601
def parse_line(self, text, fh=None):
        """"""
        match = self.ok.match(text)
        if match:
            return self._parse_result(True, match, fh)

        match = self.not_ok.match(text)
        if match:
            return self._parse_result(False, match, fh)

        if self.diagnostic.match(text):
            return Diagnostic(text)

        match = self.plan.match(text)
        if match:
            return self._parse_plan(match)

        match = self.bail.match(text)
        if match:
            return Bail(match.group("reason"))

        match = self.version.match(text)
        if match:
            return self._parse_version(match)

        return Unknown()
1209768
def parse(self, text, *, metadata=None, filename="input"):
        

        metadata = metadata or {}
        body = []
        in_metadata = True

        line_number = None

        def throw(s):
            raise BlurbError(f("Error in {filename}:{line_number}:\n{s}"))

        def finish_entry():
            nonlocal body
            nonlocal in_metadata
            nonlocal metadata
            nonlocal self

            if not body:
                throw("Blurb 'body' text must not be empty!")
            text = textwrap_body(body)
            for naughty_prefix in ("- ", "Issue #", "bpo-"):
                if text.startswith(naughty_prefix):
                    throw("Blurb 'body' can't start with " + repr(naughty_prefix) + "!")

            no_changes = metadata.get('no changes')
            section = metadata.get('section')

            if not no_changes:
                if not section:
                    throw("No 'section' specified.  You must provide one!")
                elif section not in sections:
                    throw("Invalid 'section'!  You must use one of the predefined sections.")

            bpo = None
            try:
                bpo = int(metadata.get('bpo'))
            except (TypeError, ValueError):
                throw("Invalid bpo issue number! (" + repr(bpo) + ")")

            self.append((metadata, text))
            metadata = {}
            body = []
            in_metadata = True

        for line_number, line in enumerate(text.split("\n")):
            line = line.rstrip()
            if in_metadata:
                if line.startswith('..'):
                    line = line[2:].strip()
                    name, colon, value = line.partition(":")
                    assert colon
                    name = name.strip()
                    value = value.strip()
                    if name in metadata:
                        throw("Blurb metadata sets " + repr(name) + " twice!")
                    metadata[name] = value
                    continue
                if line.startswith("#") or not line:
                    continue
                in_metadata = False

            if line == "..":
                finish_entry()
                continue
            body.append(line)

        finish_entry()