parse url for domain and homepage

        :param url: the req url
        :type url: str
        :return: (homepage, domain)
        :rtype:
961985
def get_domain_home_from_url(self, url):
        
        p = parse.urlparse(url)
        if p.netloc:
            self.domain = p.netloc
            self.homepage = '{}://{}'.format(p.scheme, p.netloc)
            return self.homepage, p.netloc
        else:
            return '', ''
466144
def get_url(self, url):
        
        _r = self.br.open(url)

        # check that we've not been redirected to the login page
        if self.br.geturl().startswith(self.AUTH_URL):
            raise AuthRequiredException
        elif self.br.geturl().startswith(self.ERROR_URL):
            raise RequestErrorException
        else:
            return _r.read()
806905
def parse_url(self):
        
        self.scheme = None
        self.resource = None
        self.host = None
        self.port = None

        if self.url is None:
            return

        scheme, url = self.url.split(":", 1)
        parsed = urlsplit(url, scheme="http")
        if parsed.hostname:
            self.host = parsed.hostname
        elif '+unix' in scheme:
            self.host = 'localhost'
        else:
            raise ValueError("Invalid hostname from: %s", self.url)

        if parsed.port:
            self.port = parsed.port

        if scheme == "ws":
            if not self.port:
                self.port = 8080
        elif scheme == "wss":
            if not self.port:
                self.port = 443
        elif scheme in ('ws+unix', 'wss+unix'):
            pass
        else:
            raise ValueError("Invalid scheme: %s" % scheme)

        if parsed.path:
            resource = parsed.path
        else:
            resource = "/"

        if '+unix' in scheme:
            self.unix_socket_path = resource
            resource = '/'

        if parsed.query:
            resource += "?" + parsed.query

        self.scheme = scheme
        self.resource = resource
347943
def get_url(url: str) -> Union[dict, int, float, str]:
        ''''''

        request = Request(url, headers={"User-Agent": "pypeerassets"})
        response = cast(HTTPResponse, urlopen(request))
        if response.status != 200:
            raise Exception(response.reason)
        return json.loads(response.read().decode())
319790
def get_request(self, url):
        
        if isinstance(url, string_types):
            url = urllib2.Request(url, headers={'Accept-encoding': 'identity'})
        return url
465709
def get(url):
    
    text = urlopen(url).read()
    soup = BeautifulSoup(text)
    return Response(text=text, soup=soup)
59690
def get_url(url, parser='html'):
    
    url = request.quote(url, safe=':/?=&')
    logger.debug('URL: %s', url)
    req = request.Request(url, headers={'User-Agent': 'foobar'})
    try:
        response = request.urlopen(req)
    except HTTPError:
        raise
    except (ssl.SSLError, URLError):
        # Some websites (like metal-archives) use older TLS versions and can
        # make the ssl module trow a VERSION_TOO_LOW error. Here we try to use
        # the older TLSv1 to see if we can fix that
        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        response = request.urlopen(req, context=context)

    response = response.read()
    if parser == 'html':
        return BeautifulSoup(response, 'html.parser', from_encoding='utf-8')
    elif parser == 'json':
        return json.loads(response)
    elif parser == 'raw':
        return response.decode()
    raise ValueError('Unrecognized parser')
834384
def _load_url(url):
    
    try:
        response = requests.get(url)
        return BytesIO(response.content)
    except IOError as ex:
        parser.error("{url} could not be loaded remotely! ({ex})".format(url=url, ex=ex))
58718
def setUrl(self, url):
        
        if isUrl(url):
            self._url = url
        else:
            raise exceptions.BadUrlException(url)
        return self
388683
def normalize_url(url: str) -> str:
    
    if url.startswith('/'):
        url = url[1:]

    if url.endswith('/'):
        url = url[:-1]

    return url
1026248
def parse_url(url, encoding=None):
    
    return url if isinstance(url, urlparse.ParseResult) else \
        urlparse.urlparse(unicode_to_str(url, encoding))
1325384
def from_url(cls, url):
        
        parsed = urlparse(url)
        return cls(
            username=parsed.username,
            password=parsed.password,
            hostname=parsed.hostname,
            port=parsed.port,
            database=parsed.path.lstrip('/'),
            # Like parse_qs, but produces a scalar per key, instead of a list:
            query_params=dict(param.split('=')
                              for param in parsed.query.split('&'))
            if parsed.query else {},
        )
140437
def parse(cls, url, default_scheme='http', encoding='utf-8'):
        ''''''
        if url is None:
            return None

        url = url.strip()
        if frozenset(url) & C0_CONTROL_SET:
            raise ValueError('URL contains control codes: {}'.format(ascii(url)))

        scheme, sep, remaining = url.partition(':')

        if not scheme:
            raise ValueError('URL missing scheme: {}'.format(ascii(url)))

        scheme = scheme.lower()

        if not sep and default_scheme:
            # Likely something like example.com/mystuff
            remaining = url
            scheme = default_scheme
        elif not sep:
            raise ValueError('URI missing colon: {}'.format(ascii(url)))

        if default_scheme and '.' in scheme or scheme == 'localhost':
            # Maybe something like example.com:8080/mystuff or
            # maybe localhost:8080/mystuff
            remaining = '{}:{}'.format(scheme, remaining)
            scheme = default_scheme

        info = URLInfo()
        info.encoding = encoding

        if scheme not in RELATIVE_SCHEME_DEFAULT_PORTS:
            info.raw = url
            info.scheme = scheme
            info.path = remaining

            return info

        if remaining.startswith('//'):
            remaining = remaining[2:]

        path_index = remaining.find('/')
        query_index = remaining.find('?')
        fragment_index = remaining.find('#')

        try:
            index_tuple = (path_index, query_index, fragment_index)
            authority_index = min(num for num in index_tuple if num >= 0)
        except ValueError:
            authority_index = len(remaining)

        authority = remaining[:authority_index]
        resource = remaining[authority_index:]

        try:
            index_tuple = (query_index, fragment_index)
            path_index = min(num for num in index_tuple if num >= 0)
        except ValueError:
            path_index = len(remaining)

        path = remaining[authority_index + 1:path_index] or '/'

        if fragment_index >= 0:
            query_index = fragment_index
        else:
            query_index = len(remaining)

        query = remaining[path_index + 1:query_index]
        fragment = remaining[query_index + 1:]

        userinfo, host = cls.parse_authority(authority)
        hostname, port = cls.parse_host(host)
        username, password = cls.parse_userinfo(userinfo)

        if not hostname:
            raise ValueError('Hostname is empty: {}'.format(ascii(url)))

        info.raw = url
        info.scheme = scheme
        info.authority = authority
        info.path = normalize_path(path, encoding=encoding)
        info.query = normalize_query(query, encoding=encoding)
        info.fragment = normalize_fragment(fragment, encoding=encoding)

        info.userinfo = userinfo
        info.username = percent_decode(username, encoding=encoding)
        info.password = percent_decode(password, encoding=encoding)

        info.host = host
        info.hostname = hostname
        info.port = port or RELATIVE_SCHEME_DEFAULT_PORTS[scheme]

        info.resource = resource

        return info
180863
def _get_url(self, url):
        
        url = str(url)
        if url != 'default' and not '://' in url:
            url = "file:" + urllib.pathname2url(url)
        return url
1069538
def _DecodeURL(self, url):
    
    if not url:
      return ''

    decoded_url = urlparse.unquote(url)
    if isinstance(decoded_url, py2to3.BYTES_TYPE):
      try:
        decoded_url = decoded_url.decode('utf-8')
      except UnicodeDecodeError as exception:
        decoded_url = decoded_url.decode('utf-8', errors='replace')
        logger.warning(
            'Unable to decode URL: {0:s} with error: {1!s}'.format(
                url, exception))

    return decoded_url
591941
def security(self, domain):
        '''
        '''
        uri = self._uris["security"].format(domain)
        return self.get_parse(uri)
340715
def get(self, url):
        
        r = requests.get(self._format_url(url), headers=self.headers, timeout=TIMEOUT)
        self._check_response(r, 200)

        return r.json()