A sane "stand alone" url parser.

    Example:

        info = chopurl(url)
551488
def chopurl(url):
    '''
    
    '''
    ret = {}
    if url.find('://') == -1:
        raise s_exc.BadUrl(':// not found in [{}]!'.format(url))

    scheme, remain = url.split('://', 1)
    ret['scheme'] = scheme.lower()

    # carve query params from the end
    if remain.find('?') != -1:
        query = {}
        remain, queryrem = remain.split('?', 1)

        for qkey in queryrem.split('&'):
            qval = None
            if qkey.find('=') != -1:
                qkey, qval = qkey.split('=', 1)

            query[qkey] = qval

        ret['query'] = query

    pathrem = ''
    slashoff = remain.find('/')
    if slashoff != -1:
        pathrem = remain[slashoff:]
        remain = remain[:slashoff]

    # detect user[:passwd]@netloc syntax
    if remain.find('@') != -1:
        user, remain = remain.rsplit('@', 1)
        if user.find(':') != -1:
            user, passwd = user.split(':', 1)
            ret['passwd'] = passwd

        ret['user'] = user

    # remain should be down to host[:port]

    # detect ipv6 [addr]:port syntax
    if remain.startswith('['):
        hostrem, portstr = remain.rsplit(':', 1)
        ret['port'] = int(portstr)
        ret['host'] = hostrem[1:-1]

    # detect ipv6 without port syntax
    elif remain.count(':') > 1:
        ret['host'] = remain

    # regular old host or host:port syntax
    else:

        if remain.find(':') != -1:
            remain, portstr = remain.split(':', 1)
            ret['port'] = int(portstr)

        ret['host'] = remain

    ret['path'] = pathrem
    return ret
975633
def get_url_for_get(url, parameters=None):
        # type: (str, Optional[Dict]) -> str
        
        spliturl = urlsplit(url)
        getparams = OrderedDict(parse_qsl(spliturl.query))
        if parameters is not None:
            getparams.update(parameters)
        spliturl = spliturl._replace(query=urlencode(getparams))
        return urlunsplit(spliturl)
867502
def parse_source(info):
    

    if "extractor_key" in info:
        source = info["extractor_key"]
        lower_source = source.lower()

        for key in SOURCE_TO_NAME:
            lower_key = key.lower()
            if lower_source == lower_key:
                source = SOURCE_TO_NAME[lower_key]

        if source != "Generic":
            return source

    if "url" in info and info["url"] is not None:
        p = urlparse(info["url"])
        if p and p.netloc:
            return p.netloc

    return "Unknown"
59690
def get_url(url, parser='html'):
    
    url = request.quote(url, safe=':/?=&')
    logger.debug('URL: %s', url)
    req = request.Request(url, headers={'User-Agent': 'foobar'})
    try:
        response = request.urlopen(req)
    except HTTPError:
        raise
    except (ssl.SSLError, URLError):
        # Some websites (like metal-archives) use older TLS versions and can
        # make the ssl module trow a VERSION_TOO_LOW error. Here we try to use
        # the older TLSv1 to see if we can fix that
        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        response = request.urlopen(req, context=context)

    response = response.read()
    if parser == 'html':
        return BeautifulSoup(response, 'html.parser', from_encoding='utf-8')
    elif parser == 'json':
        return json.loads(response)
    elif parser == 'raw':
        return response.decode()
    raise ValueError('Unrecognized parser')
323999
def urlunparse(parts):
    """"""
    scheme, netloc, path, params, query, fragment = parts

    # Avoid encoding the windows drive letter colon
    if RE_DRIVE_LETTER_PATH.match(path):
        quoted_path = path[:3] + parse.quote(path[3:])
    else:
        quoted_path = parse.quote(path)

    return parse.urlunparse((
        parse.quote(scheme),
        parse.quote(netloc),
        quoted_path,
        parse.quote(params),
        parse.quote(query),
        parse.quote(fragment)
    ))
238384
def url(self, name):
        """"""
        scheme = 'http'
        path = self._prepend_name_prefix(name)
        query = ''
        fragment = ''
        url_tuple = (scheme, self.netloc, path, query, fragment)
        return urllib.parse.urlunsplit(url_tuple)
358222
def urlunparse(components):
    """"""
    scheme, netloc, url, params, query, fragment, _coerce_result = (
                                                  _coerce_args(*components))
    if params:
        url = "%s;%s" % (url, params)
    return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment)))
1325384
def from_url(cls, url):
        
        parsed = urlparse(url)
        return cls(
            username=parsed.username,
            password=parsed.password,
            hostname=parsed.hostname,
            port=parsed.port,
            database=parsed.path.lstrip('/'),
            # Like parse_qs, but produces a scalar per key, instead of a list:
            query_params=dict(param.split('=')
                              for param in parsed.query.split('&'))
            if parsed.query else {},
        )
747917
def parse(url):
    
    result = urlparse.urlparse(nstr(url))

    path = result.scheme + '://' + result.netloc
    if result.path:
        path += result.path

    query = {}

    # extract the python information from the query
    if result.query:
        url_query = urlparse.parse_qs(result.query)
        for key, value in url_query.items():
            if type(value) == list and len(value) == 1:
                value = value[0]

            query[key] = value

    return path, query, result.fragment
383793
def explicit_rel_links(self, rels=('homepage', 'download')):
        """"""
        for match in self._rel_re.finditer(self.content):
            found_rels = match.group(1).lower().split()
            for rel in rels:
                if rel in found_rels:
                    break
            else:
                continue
            match = self._href_re.search(match.group(0))
            if not match:
                continue
            url = match.group(1) or match.group(2) or match.group(3)
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self)
33718
def parse_url(url, options)
      url.gsub!(/\s/, '')
      url.gsub!(/\?.*/, '') if options.strip_query?
      url.gsub!(/\#.*/, '') if options.strip_anchor?
      Addressable::URI.parse(url)
    end
619369
def split_url(url)
      str = split_get_last(split_get_last(url, '/'), '?')
      str = split_get_last(str, '&', -1)
      str
    end
403372
function paramsParser(url, rule) {
    var matches = rule.regexp.exec(url).slice(1);
    var keys = rule.keys;
    var params = {};

    for (var i = 0; i < keys.length; i++) {
      params[keys[i]] = matches[i];
    }

    return params;
  }
75245
def site(parser, token):
    '''
    
    '''
    node = url(parser, token)
    return SiteUrlNode(
        view_name=node.view_name,
        args=node.args,
        kwargs=node.kwargs,
        asvar=node.asvar
    )
1188080
function parse(url) {
  ['?', '#'].forEach(function(separator) {
    var index = url.indexOf(separator);
    if (index !== -1) url = url.substring(0, index);
  });
  return url;
}
843628
def clean_url(url, base_url=None):
    """"""
    parsed_url = urlparse(url)

    fragment = '{url.fragment}'.format(url=parsed_url)
    if fragment:
        url = url.split(fragment)[0]

    # Identify internal URLs and fix their format
    netloc = '{url.netloc}'.format(url=parsed_url)
    if base_url is not None and not netloc:
        parsed_base = urlparse(base_url)
        split_base = '{url.scheme}://{url.netloc}{url.path}/'.format(url=parsed_base)
        url = urljoin(split_base, url)
        netloc = '{url.netloc}'.format(url=urlparse(url))

    if 'www.' in netloc:
        url = url.replace(netloc, netloc.replace('www.', ''))
    return url.rstrip(string.punctuation)
132198
def find_links (url_data, callback, tags):
    
    # construct parser object
    handler = linkparse.LinkFinder(callback, tags)
    parser = htmlsax.parser(handler)
    if url_data.charset:
        parser.encoding = url_data.charset
    handler.parser = parser
    # parse
    try:
        parser.feed(url_data.get_content())
        parser.flush()
    except linkparse.StopParse as msg:
        log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
        pass
    # break cyclic dependencies
    handler.parser = None
    parser.handler = None