Parse source into a Python AST, taking care of encoding.
962202
def parse_ast(source, filename=None):
    """"""
    if isinstance(source, text_type) and sys.version_info[0] == 2:
        # ast.parse() on Python 2 doesn't like encoding declarations
        # in Unicode strings
        source = CODING_COOKIE_RE.sub(r'\1', source, 1)
    return ast.parse(source, filename or '<unknown>')
1339490
def RawParserUnparserFactory(parser_name, parse_callable, *unparse_callables):
    

    def build_unparse(f):
        @wraps(f)
        def unparse(self, source, *a, **kw):
            node = parse_callable(source)
            return f(node, *a, **kw)
        # a dumb and lazy docstring replacement
        unparse.__doc__ = f.__doc__.replace(
            'ast\n        The AST ',
            'source\n        The source ',
        )
        return unparse

    def build_parse(f):
        @wraps(f)
        def parse(self, source):
            return f(source)
        parse.__name__ = parser_name
        parse.__qualname__ = parser_name
        return parse

    callables = {f.__name__: build_unparse(f) for f in unparse_callables}
    callables['__call__'] = build_parse(parse_callable)
    callables['__module__'] = PKGNAME
    return type(parser_name, (object,), callables)()
201955
def load(cls, filename, encoding):
        """"""
        if encoding is None:
            encoding = cls.encoding

        with io.open(filename, 'r', encoding=encoding) as fd:
            source = fd.read()

        if cls.normalize_newlines:
            source = source.replace('\r\n', '\n').replace('\r', '\n')
        return cls.loads(source)
808015
def sourcelines(self):
        """"""
        # get sourcecode from loader or file
        source = None
        if self.loader is not None:
            try:
                if hasattr(self.loader, 'get_source'):
                    source = self.loader.get_source(self.module)
                elif hasattr(self.loader, 'get_source_by_code'):
                    source = self.loader.get_source_by_code(self.code)
            except Exception:
                # we munch the exception so that we don't cause troubles
                # if the loader is broken.
                pass

        if source is None:
            try:
                f = open(self.filename)
            except IOError:
                return []
            try:
                source = f.read()
            finally:
                f.close()

        # already unicode?  return right away
        if isinstance(source, str):
            return source.splitlines()

        # yes. it should be ascii, but we don't want to reject too many
        # characters in the debugger if something breaks
        charset = 'utf-8'
        if source.startswith(UTF8_COOKIE):
            source = source[3:]
        else:
            for idx, match in enumerate(_line_re.finditer(source)):
                match = _line_re.search(match.group())
                if match is not None:
                    charset = match.group(1)
                    break
                if idx > 1:
                    break

        # on broken cookies we fall back to utf-8 too
        try:
            codecs.lookup(charset)
        except LookupError:
            charset = 'utf-8'

        return source.decode(charset, 'replace').splitlines()
908725
def tokenize_asdl(buf):
    """"""
    for lineno, line in enumerate(buf.splitlines(), 1):
        for m in re.finditer(r'\s*(\w+|--.*|.)', line.strip()):
            c = m.group(1)
            if c[0].isalpha():
                # Some kind of identifier
                if c[0].isupper():
                    yield Token(TokenKind.ConstructorId, c, lineno)
                else:
                    yield Token(TokenKind.TypeId, c, lineno)
            elif c[:2] == '--':
                # Comment
                break
            else:
                # Operators
                try:
                    op_kind = TokenKind.operator_table[c]
                except KeyError:
                    raise ASDLSyntaxError('Invalid operator %s' % c, lineno)
                yield Token(op_kind, c, lineno)
1232731
def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
          parse_int=None, parse_constant=None, object_pairs_hook=None):
    """"""

    assert cls is None, 'Custom decoders are not supported'

    if sys.version_info[0] < 3:
        decodable_type = type('')
    else:
        decodable_type = type(b'')
    if isinstance(s, decodable_type):
        encoding = encoding or 'utf-8'
        s = s.decode(encoding)

    if not s:
        raise ValueError('Empty strings are not legal JSON5')
    parser = Parser(s, '<string>')
    ast, err, newpos = parser.parse()
    if err:
        raise ValueError(err)

    def _fp_constant_parser(s):
        return float(s.replace('Infinity', 'inf').replace('NaN', 'nan'))

    if object_pairs_hook:
        dictify = object_pairs_hook
    elif object_hook:
        dictify = lambda pairs: object_hook(dict(pairs))
    else:
        dictify = dict

    parse_float = parse_float or float
    parse_int = parse_int or int
    parse_constant = parse_constant or _fp_constant_parser

    return _walk_ast(ast, dictify, parse_float, parse_int, parse_constant)
722349
def parse(self, text):
        """"""
        self.expr = text
        try:
            out = ast.parse(text)
        except SyntaxError:
            self.raise_exception(None, msg='Syntax Error', expr=text)
        except:
            self.raise_exception(None, msg='Runtime Error', expr=text)

        return out
106628
def parse_options(source):
    """"""
    options = {}
    tokens = [t.strip() for t in source.split('=')]

    name = tokens[0]
    for token in tokens[1:-1]:
        value, next_name = token.rsplit(' ', 1)
        options[name.strip()] = value
        name = next_name
    options[name.strip()] = tokens[-1].strip()
    return options
930537
def load_source(self):
    """"""
    if self.filename in self.STDIN_NAMES:
      self.filename = 'stdin'
      self.source = pycodestyle.stdin_get_value()
    else:
      with pep257.tokenize_open(self.filename) as fd:
        self.source = fd.read()
284533
def _to_unicode(s):
    
    try:
        return s.encode('ascii')
    except (UnicodeError, AttributeError):
        try:
            return s.decode('utf-8')
        except UnicodeError:
            return s
867502
def parse_source(info):
    

    if "extractor_key" in info:
        source = info["extractor_key"]
        lower_source = source.lower()

        for key in SOURCE_TO_NAME:
            lower_key = key.lower()
            if lower_source == lower_key:
                source = SOURCE_TO_NAME[lower_key]

        if source != "Generic":
            return source

    if "url" in info and info["url"] is not None:
        p = urlparse(info["url"])
        if p and p.netloc:
            return p.netloc

    return "Unknown"
846769
def parse(code, mode='exec', **exception_kwargs):
    """"""

    try:
        return _ast_util.parse(code, '<unknown>', mode)
    except Exception:
        raise exceptions.SyntaxException(
                    "(%s) %s (%r)" % (
                        compat.exception_as().__class__.__name__,
                        compat.exception_as(),
                        code[0:50]
                    ), **exception_kwargs)
1286579
def parse(self, data):
        """"""

        ASNlist = []
        for line in data.splitlines()[1:]:
            line = plain_str(line)
            if "|" not in line:
                continue
            asn, ip, desc = [elt.strip() for elt in line.split('|')]
            if asn == "NA":
                continue
            asn = "AS%s" % asn
            ASNlist.append((ip, asn, desc))
        return ASNlist
383167
def parse_token_contents(parser, token):
    
    bits = token.split_contents()
    tag = bits.pop(0)
    args = []
    kwargs = {}
    asvar = None
    if len(bits) >= 2 and bits[-2] == "as":
        asvar = bits[-1]
        bits = bits[:-2]
    for bit in bits:
        match = kwarg_re.match(bit)
        if not match:
            raise TemplateSyntaxError('Malformed arguments to tag "{}"'.format(tag))
        name, value = match.groups()
        if name:
            kwargs[name] = parser.compile_filter(value)
        else:
            args.append(parser.compile_filter(value))
    return {"tag": tag, "args": args, "kwargs": kwargs, "asvar": asvar}
343018
def decode_raw_stream(self, text, decode_raw, known_encoding, filename):
        
        if isinstance(text, compat.text_type):
            m = self._coding_re.match(text)
            encoding = m and m.group(1) or known_encoding or 'ascii'
            return encoding, text

        if text.startswith(codecs.BOM_UTF8):
            text = text[len(codecs.BOM_UTF8):]
            parsed_encoding = 'utf-8'
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m is not None and m.group(1) != 'utf-8':
                raise exceptions.CompileException(
                                "Found utf-8 BOM in file, with conflicting "
                                "magic encoding comment of '%s'" % m.group(1),
                                text.decode('utf-8', 'ignore'),
                                0, 0, filename)
        else:
            m = self._coding_re.match(text.decode('utf-8', 'ignore'))
            if m:
                parsed_encoding = m.group(1)
            else:
                parsed_encoding = known_encoding or 'ascii'

        if decode_raw:
            try:
                text = text.decode(parsed_encoding)
            except UnicodeDecodeError:
                raise exceptions.CompileException(
                        "Unicode decode operation of encoding '%s' failed" %
                        parsed_encoding,
                        text.decode('utf-8', 'ignore'),
                        0, 0, filename)

        return parsed_encoding, text
1221554
def inline_parse(text, verbose=False):
    """"""
    if isinstance(text, str):
        text = text.decode('utf-8')

    parsed = parse(text, verbose=verbose)

    shift = 0
    for quantity in parsed:
        index = quantity.span[1] + shift
        to_add = u' {' + unicode(quantity) + u'}'
        text = text[0:index] + to_add + text[index:]
        shift += len(to_add)

    return text
627150
def source_to_code(self, nodes, path, *, _optimize=-1):
        """"""
        if not isinstance(nodes, ast.Module):
            nodes = ast.parse(nodes, self.path)
        if self._markdown_docstring:
            nodes = update_docstring(nodes)
        return super().source_to_code(
            ast.fix_missing_locations(self.visit(nodes)), path, _optimize=_optimize
        )