Perform intensive care on URLs, see `urlnormalizer`.
498180
def clean_text(self, url, **kwargs):
        """"""
        try:
            return normalize_url(url)
        except UnicodeDecodeError:
            log.warning("Invalid URL: %r", url)
227342
function updateUrlParameter (uri, key, value) {
	// remove the hash part before operating on the uri
	var
		i = uri.indexOf('#'),
		hash = i === -1 ? '' : uri.substr(i)
		;

	uri = i === -1 ? uri : uri.substr(0, i);

	var
		re = new RegExp("([?&])" + key + "=.*?(&|$)", "i"),
		separator = uri.indexOf('?') !== -1 ? "&" : "?"
		;

	if (!value) {
		// remove key-value pair if value is empty
		uri = uri.replace(new RegExp("([?&]?)" + key + "=[^&]*", "i"), '');

		if (uri.slice(-1) === '?') {
			uri = uri.slice(0, -1);
		}
		// replace first occurrence of & by ? if no ? is present

		if (uri.indexOf('?') === -1) {
			uri = uri.replace(/&/, '?');
		}

	} else if (uri.match(re)) {
		uri = uri.replace(re, '$1' + key + "=" + value + '$2');
	} else {
		uri = uri + separator + key + "=" + value;
	}
	return uri + hash;
}
1024607
def urlretrieve(url, dest, write_mode="w"):
    """"""
    response = urllib2.urlopen(url)
    mkdir_recursive(os.path.dirname(dest))
    with open(dest, write_mode) as f:
        f.write(response.read())
        f.close()
731706
def normalize(url, strip=False):
    ""
    if strip:
        p = _urltools.parse(url)
        url = p.scheme + '://' + p.subdomain + p.domain + p.path
    return _urltools.normalize(url)
46022
def urlsplit(url):
    
    proto, rest = url.split(':', 1)
    host = ''
    if rest[:2] == '//':
        host, rest = rest[2:].split('/', 1)
        rest = '/' + rest
    return proto, host, rest
1111677
def loadu(self, url, **kwargs):
        
        return self.load(self.open(url, **kwargs), **kwargs)
1215144
public URLNormalizer lowerCaseSchemeHost() {
        URL u = toURL();
        url = Pattern.compile(u.getProtocol(),
                Pattern.CASE_INSENSITIVE).matcher(url).replaceFirst(
                u.getProtocol().toLowerCase());
        url = Pattern.compile(u.getHost(),
                Pattern.CASE_INSENSITIVE).matcher(url).replaceFirst(
                u.getHost().toLowerCase());
        return this;
    }
1282212
def replace_urls(status):
    '''
    
    '''
    text = status.text

    if not has_url(status):
        return text

    urls = [(e['indices'], e['expanded_url']) for e in status.entities['urls']]
    urls.sort(key=lambda x: x[0][0], reverse=True)

    for (start, end), url in urls:
        text = text[:start] + url + text[end:]

    return text
209647
def _build_urlmapping(urls, strict_slashes=False, **kwargs):
    

    rules = _build_rules(urls)
    return Map(rules=list(rules), strict_slashes=strict_slashes, **kwargs)
768997
def normalize_url(node):
    """"""
    if not node:
        node = DEFAULT_NODE
    elif '://' not in node:
        node = '//{}'.format(node)
    parts = urlparse(node, scheme='http', allow_fragments=False)
    port = parts.port if parts.port else _get_default_port(parts.scheme)
    netloc = '{}:{}'.format(parts.hostname, port)
    return urlunparse((parts.scheme, netloc, parts.path, '', '', ''))
1161923
def prepare_url(self, uri, kwargs):
        
        params = dict()
        for key in kwargs:
            if key in ('include', 'exclude', 'fields'):
                params.update({
                    key: ','.join(kwargs.get(key))
                })
            elif key in ('search', 'kind'):
                params.update({
                    key: kwargs.get(key)
                })

        if params:
            params = urlencode(params)
            uri = '%s?%s' % (uri, params)

        return uri
1148890
function(url) {
  var done = _onImportScript(url)
  loadScript(
    url,
    function runScript(err, code) {
      if (err) return done(err)
      executeNormal(code, url, done)
    }
  )
}
1211163
def normalize_url(href) #:nodoc:
      uri = URI.parse(href)
      normalized_url = []
      normalized_url << "#{uri.scheme}://" if uri.scheme
      normalized_url << uri.host if uri.host
      normalized_url << ":#{uri.port}" if uri.port && ![80,443].include?(uri.port)
      normalized_url << uri.path if uri.path
      normalized_url << "?#{uri.query}" if uri.query
      normalized_url.join
    end
309033
def urlopen(self, method, url, redirect=True, **kw):
        ""
        u = parse_url(url)

        if u.scheme == "http":
            # For proxied HTTPS requests, httplib sets the necessary headers
            # on the CONNECT to the proxy. For HTTP, we'll definitely
            # need to set 'Host' at the very least.
            headers = kw.get('headers', self.headers)
            kw['headers'] = self._set_proxy_headers(url, headers)

        return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
146676
def post_loader(*decorator_args, serializer):
    
    def wrapped(fn):
        @wraps(fn)
        def decorated(*args, **kwargs):
            return fn(*serializer.load(request.get_json()))
        return decorated

    if decorator_args and callable(decorator_args[0]):
        return wrapped(decorator_args[0])
    return wrapped
1069015
def from_url(cls, url, filename=None, chunk_size=CHUNK_SIZE):
        
        pipe = _WebPipe(url, chunk_size=chunk_size)
        if filename is None:
            filename = pipe.name

        return cls(pipe, filename, chunk_size)
279958
def expand(self, url):
        """"""
        url = self.clean_url(url)
        response = self._get(url)
        if response.ok:
            return response.url
        raise ExpandingErrorException