A slightly safer version of ``urlib2.urlopen`` which prevents redirection
    and ensures the URL isn't attempting to hit a blacklisted IP range.
555634
def urlopen(url, method=None, params=None, data=None, json=None,
            headers=None, allow_redirects=False, timeout=30,
            verify_ssl=True, user_agent=None):
    
    if user_agent is not None:
        warnings.warn('user_agent is no longer used with safe_urlopen')

    session = build_session()

    kwargs = {}

    if json:
        kwargs['json'] = json
        if not headers:
            headers = {}
        headers.setdefault('Content-Type', 'application/json')

    if data:
        kwargs['data'] = data

    if params:
        kwargs['params'] = params

    if headers:
        kwargs['headers'] = headers

    if method is None:
        method = 'POST' if (data or json) else 'GET'

    try:
        response = session.request(
            method=method,
            url=url,
            allow_redirects=allow_redirects,
            timeout=timeout,
            verify=verify_ssl,
            **kwargs
        )
    # Our version of requests does not transform ZeroReturnError into an
    # appropriately generically catchable exception
    except ZeroReturnError as exc:
        import sys
        exc_tb = sys.exc_info()[2]
        six.reraise(SSLError, exc, exc_tb)
        del exc_tb

    # requests' attempts to use chardet internally when no encoding is found
    # and we want to avoid that slow behavior
    if not response.encoding:
        response.encoding = 'utf-8'

    return response
4598
def url_fix_host (urlparts):
    """"""
    if not urlparts[1]:
        urlparts[2] = urllib.unquote(urlparts[2])
        return False
    userpass, netloc = urllib.splituser(urlparts[1])
    if userpass:
        userpass = urllib.unquote(userpass)
    netloc, is_idn = idna_encode(urllib.unquote(netloc).lower())
    # a leading backslash in path causes urlsplit() to add the
    # path components up to the first slash to host
    # try to find this case...
    i = netloc.find("\\")
    if i != -1:
        # ...and fix it by prepending the misplaced components to the path
        comps = netloc[i:] # note: still has leading backslash
        if not urlparts[2] or urlparts[2] == '/':
            urlparts[2] = comps
        else:
            urlparts[2] = "%s%s" % (comps, urllib.unquote(urlparts[2]))
        netloc = netloc[:i]
    else:
        # a leading ? in path causes urlsplit() to add the query to the
        # host name
        i = netloc.find("?")
        if i != -1:
            netloc, urlparts[3] = netloc.split('?', 1)
        # path
        urlparts[2] = urllib.unquote(urlparts[2])
    if userpass:
        # append AT for easy concatenation
        userpass += "@"
    else:
        userpass = ""
    if urlparts[0] in default_ports:
        dport = default_ports[urlparts[0]]
        host, port = splitport(netloc, port=dport)
        if host.endswith("."):
            host = host[:-1]
        if port != dport:
            host = "%s:%d" % (host, port)
        netloc = host
    urlparts[1] = userpass+netloc
    return is_idn
1326745
def __ip_addr(addr, address_family=socket.AF_INET):
    '''
    
    '''
    mask_max = '32'
    if address_family == socket.AF_INET6:
        mask_max = '128'

    try:
        if '/' not in addr:
            addr = '{addr}/{mask_max}'.format(addr=addr, mask_max=mask_max)
    except TypeError:
        return False

    ip, mask = addr.rsplit('/', 1)

    # Verify that IP address is valid
    try:
        socket.inet_pton(address_family, ip)
    except socket.error:
        return False

    # Verify that mask is valid
    try:
        mask = int(mask)
    except ValueError:
        return False
    else:
        if not 1 <= mask <= int(mask_max):
            return False

    return True
483485
def prune_urls(url_set, start_url, allowed_list, ignored_list):
    """"""
    result = set()

    for url in url_set:
        allowed = False
        for allow_url in allowed_list:
            if url.startswith(allow_url):
                allowed = True
                break

        if not allowed:
            continue

        ignored = False
        for ignore_url in ignored_list:
            if url.startswith(ignore_url):
                ignored = True
                break

        if ignored:
            continue

        prefix, suffix = (url.rsplit('.', 1) + [''])[:2]
        if suffix.lower() in IGNORE_SUFFIXES:
            continue

        result.add(url)

    return result
1233276
def urlopen(url, session, referrer=None, max_content_bytes=None,
            timeout=ConnectionTimeoutSecs, raise_for_status=True,
            stream=False, data=None, useragent=UserAgent):
    """"""
    out.debug(u'Open URL %s' % url)
    headers = {'User-Agent': useragent}
    if referrer:
        headers['Referer'] = referrer
    out.debug(u'Sending headers %s' % headers, level=3)
    out.debug(u'Sending cookies %s' % session.cookies)
    kwargs = {
        "headers": headers,
        "timeout": timeout,
    }
    if hasattr(requests, 'adapters'):
        # requests >= 1.0
        kwargs["stream"] = stream
    else:
        # requests << 1.0
        kwargs["prefetch"] = not stream
        kwargs["config"] = {"max_retries": MaxRetries}
    if data is None:
        func = session.get
    else:
        kwargs['data'] = data
        func = session.post
        out.debug(u'Sending POST data %s' % data, level=3)
    try:
        req = func(url, **kwargs)
        out.debug(u'Response cookies: %s' % req.cookies)
        check_content_size(url, req.headers, max_content_bytes)
        if raise_for_status:
            req.raise_for_status()
        return req
    except requests.exceptions.RequestException as err:
        msg = 'URL retrieval of %s failed: %s' % (url, err)
        raise IOError(msg)
1064930
def _ensure_html_response(url, session):
    # type: (str, PipSession) -> None
    
    scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
    if scheme not in {'http', 'https'}:
        raise _NotHTTP()

    resp = session.head(url, allow_redirects=True)
    resp.raise_for_status()

    _ensure_html_header(resp)
715623
def safe_url(url):
    """"""
    parsed = urlparse(url)
    if parsed.password is not None:
        pwd = ':%s@' % parsed.password
        url = url.replace(pwd, ':*****@')
    return url
1213831
def inject_into_urllib3():
    
    util.ssl_.SSLContext = SecureTransportContext
    util.HAS_SNI = HAS_SNI
    util.ssl_.HAS_SNI = HAS_SNI
    util.IS_SECURETRANSPORT = True
    util.ssl_.IS_SECURETRANSPORT = True
570910
def urlopen(url, headers={}, data=None, retries=RETRIES, timeout=TIMEOUT):
    '''
    '''
    headers_merged = default_headers.copy()
    for key in headers.keys():
        headers_merged[key] = headers[key]
    opener = urllib.request.build_opener(ForbiddenHandler)
    opener.addheaders = [(k, v) for k,v in headers_merged.items()]

    for i in range(retries):
        try:
            req = opener.open(url, data=data, timeout=timeout)
            encoding = req.headers.get('Content-encoding')
            req.data = req.read()
            if encoding == 'gzip':
                req.data = gzip.decompress(req.data)
            elif encoding == 'deflate':
                req.data = zlib.decompress(req.data, -zlib.MAX_WBITS)
            return req
        except OSError:
            logger.error(traceback.format_exc())
            
        except:
            logger.error(traceback.format_exc())
            
    return None
948886
def validate_url(cls, url):
        

        if url is None:
            return None

        url = url.strip()
        while url.endswith('/'):
            url = url[:-1]
        return url
649893
def check_url(url):
        '''
        

        '''
        request = urllib2.Request(url)
        try:
            response = urlopen(request)
            return True, response.code
        except urllib2.HTTPError as e:
            return False, e.code
1052341
def validated_url(url):
    """"""
    try:
        u = urlparse(url)
        u.port  # Trigger 'invalid port' exception
    except Exception:
        raise error.URLError(url)
    else:
        if not u.scheme or not u.netloc:
            raise error.URLError(url)
        return url
801440
def validate(url, schemes=None, tlds=None, private=None, local=None,
             credentials=None):
    '''
    
    '''
    url = url.strip()

    private = config_for(private, 'URLS_ALLOW_PRIVATE')
    local = config_for(local, 'URLS_ALLOW_LOCAL')
    credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS')
    schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES')
    tlds = config_for(tlds, 'URLS_ALLOWED_TLDS')

    match = URL_REGEX.match(url)
    if not match:
        error(url)

    scheme = (match.group('scheme') or '').lower()
    if scheme and scheme not in schemes:
        error(url, 'Invalid scheme {0}'.format(scheme))

    if not credentials and match.group('credentials'):
        error(url, 'Credentials in URL are not allowed')

    tld = match.group('tld')
    if tld and tld not in tlds and tld.encode('idna') not in tlds:
        error(url, 'Invalid TLD {0}'.format(tld))

    ip = match.group('ipv6') or match.group('ipv4')
    if ip:
        try:
            ip = IPAddress(ip)
        except AddrFormatError:
            error(url)
        if ip.is_multicast():
            error(url, '{0} is a multicast IP'.format(ip))
        elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask():
            error(url, '{0} is a mask IP'.format(ip))

    if not local:
        if ip and ip.is_loopback() or match.group('localhost'):
            error(url, 'is a local URL')

    if not private and ip and ip.is_private():
        error(url, 'is a private URL')

    return url
63895
def _proxy(self, url, urlparams=None):
        
        for k,v in request.params.iteritems():
            urlparams[k]=v
        query = urlencode(urlparams)
        full_url = url
        if query:
            if not full_url.endswith("?"):
                full_url += "?"
            full_url += query

        # build the request with its headers
        req = urllib2.Request(url=full_url)
        for header in request.headers:
            if header.lower() == "host":
                req.add_header(header, urlparse.urlparse(url)[1])
            else:
                req.add_header(header, request.headers[header])
        res = urllib2.urlopen(req)

        # add response headers
        i = res.info()
        response.status = res.code
        got_content_length = False
        for header in i:
            # We don't support serving the result as chunked
            if header.lower() == "transfer-encoding":
                continue
            if header.lower() == "content-length":
                got_content_length = True
            response.headers[header] = i[header]

        # return the result
        result = res.read()
        res.close()

        #if not got_content_length:
        #    response.headers['content-length'] = str(len(result))
        return result
479285
def urlopen(link):
    
    try:
        return urllib2.urlopen(link)
    except urllib2.URLError:
        pass
    except ValueError:
        return ""
    except KeyboardInterrupt:
        print("")
        raise SystemExit()
504132
def netloc_no_www(url):
    
    ext = tldextract.extract(url)
    if ext.subdomain and ext.subdomain != 'www':
        return '%s.%s.%s' % (ext.subdomain, ext.domain, ext.tld) 
    else:
        return '%s.%s' % (ext.domain, ext.tld)
85673
def _unapply_xheaders(self) -> None:
        
        self.remote_ip = self._orig_remote_ip
        self.protocol = self._orig_protocol