This function uses several methods to extract URLs from 'thing', which can be a string or raw bytes.
        If you supply the base URL, it will attempt to use it with urljoin to create full URLs from relative paths.
425098
def find_urls(thing, base_url=None, mimetype=None, log=False):
    

    if log:
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s %(name)s.%(funcName)s +%(lineno)s: %(levelname)-8s %(message)s')
        logger = logging.getLogger(__name__)

    # Convert "thing" to bytes if it is a string.
    try:
        if isinstance(thing, str):
            thing = thing.encode(encoding='ascii', errors='ignore')
    except:
        if log:
            logger.exception('Unable to convert thing to bytes.')
        return []

    # Store any URLs we find in the bytes.
    all_urls = []

    # Continue if we have bytes.
    if isinstance(thing, bytes):

        # Return an empty list if we failed to get the mimetype.
        try:
            if not mimetype:
                mimetype = magic.from_buffer(thing)
        except:
            if log:
                logger.exception('Unable to get mimetype from the bytes buffer.')
            return []

        mimetype = mimetype.lower()

        # If the bytes are HTML...
        if 'html' in mimetype:
            try:
                all_urls += _html_find_urls(thing, mimetype, base_url)
            except:
                if log:
                    logger.exception('Error when finding HTML URLs.')

        # If the bytes are a PDF...
        elif 'pdf' in mimetype:
            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding PDF URLs.')

        # If the bytes are an RFC 822 e-mail...
        elif 'rfc 822' in mimetype:
            return []

        # If the bytes are ASCII or Unicode text...
        elif 'ascii' in mimetype or 'text' in mimetype:
            try:
                all_urls += _html_find_urls(thing, mimetype, base_url)
            except:
                if log:
                    logger.exception('Error when finding ASCII/HTML URLs.')

            try:
                all_urls += _ascii_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding ASCII URLs.')

            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding ASCII/PDF URLs.')

        # If the bytes are anything else...
        else:

            # Try to treat the bytes as a PDF and find URLs.
            try:
                all_urls += _pdf_find_urls(thing, mimetype)
            except:
                if log:
                    logger.exception('Error when finding unknown/PDF URLs.')

            # If we don't know how to handle this mimetype, the bytes are likely just "data".
            # In that case, we don't want to find all possible ASCII URLs, as it will result
            # in a lot of bad URLs. Try to treat the bytes as ASCII and find URLs.
            try:
                all_urls += _ascii_find_urls(thing, mimetype, extra_tokens=False)
            except:
                if log:
                    logger.exception('Error when finding unknown/ASCII URLs.')

    # Make sure we only have valid URLs.
    valid_urls = []
    for url in list(set(all_urls)):
        try:
            # If the URL is valid as-is, just add it to the list.
            if is_valid(url):
                valid_urls.append(url)

            # The URL is not valid. If we were given a base URL, try joining them and checking if the result is valid.
            elif base_url:
                joined_url = urljoin(base_url, url)
                if is_valid(joined_url):
                    valid_urls.append(joined_url)
        except:
            pass

    # For the edge cases of HTML files where we didn't find any URLs, treat it as an ASCII file
    # and re-find any URLs that way.
    if not valid_urls and 'html' in mimetype:
        try:
            for url in _ascii_find_urls(thing, mimetype):
                # If the URL is valid as-is, just add it to the list.
                if is_valid(url):
                    valid_urls.append(url)

                # The URL is not valid. If we were given a base URL, try joining them and checking if the result is valid.
                elif base_url:
                    joined_url = urljoin(base_url, url)
                    if is_valid(joined_url):
                        valid_urls.append(joined_url)
        except:
            pass

    # Return the valid URLs in ASCII form.
    ascii_urls = []
    for url in valid_urls:
        try:
            if isinstance(url, str):
                ascii_urls.append(url)

            if isinstance(url, bytes):
                ascii_urls.append(url.decode('ascii', errors='ignore'))
        except:
            pass

    # Check if any of the URLs are Proofpoint URLs and try to decode them.
    for url in ascii_urls[:]:
        if 'urldefense.proofpoint.com/v2/url' in url:
            try:
                query_u=parse_qs(urlparse(url).query)['u'][0]
                decoded_url = query_u.replace('-3A', ':').replace('_', '/').replace('-2D', '-')
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Proofpoint URL: {}'.format(url))

    # Check if any of the URLs are Outlook safelinks and try to decode them.
    for url in ascii_urls[:]:
        if 'safelinks.protection.outlook.com' in url:
            try:
                query_url=parse_qs(urlparse(url).query)['url'][0]
                decoded_url = urllib.parse.unquote(query_url)
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Outlook safelinks URL: {}'.format(url))

    # Check if any of the URLs are Google redirection URLs and try to decode them.
    for url in ascii_urls[:]:
        if 'www.google.com/url?' in url:
            try:
                query_url=parse_qs(urlparse(url).query)['q'][0]
                decoded_url = urllib.parse.unquote(query_url)
                if is_valid(decoded_url):
                    ascii_urls.append(decoded_url)
            except:
                if log:
                    logger.exception('Error decoding Google redirection URL: {}'.format(url))

    # Add an unquoted version of each URL to the list.
    for url in ascii_urls[:]:
        ascii_urls.append(urllib.parse.unquote(url))

    # Add http:// to the beginning of each URL if it isn't there already. This lets us properly
    # catch URLs that may not have the scheme on the front of them.
    ascii_urls = ['http://' + u if not u.lower().startswith('http') and not u.lower().startswith('ftp') else u for u in ascii_urls]

    # Remove any trailing "/" from the URLs so that they are consistent with how they go in CRITS.
    ascii_urls = [u[:-1] if u.endswith('/') else u for u in ascii_urls]

    return sorted(list(set(ascii_urls)))
268405
def get(url):
    
    url = text_type(url)
    if url.find("://") == -1:
        Log.error("{{url}} must have a prototcol (eg http://) declared", url=url)

    base = URL("")
    if url.startswith("file://") and url[7] != "/":
        if os.sep=="\\":
            base = URL("file:///" + os.getcwd().replace(os.sep, "/").rstrip("/") + "/.")
        else:
            base = URL("file://" + os.getcwd().rstrip("/") + "/.")
    elif url[url.find("://") + 3] != "/":
        Log.error("{{url}} must be absolute", url=url)

    phase1 = _replace_ref(wrap({"$ref": url}), base)  # BLANK URL ONLY WORKS IF url IS ABSOLUTE
    try:
        phase2 = _replace_locals(phase1, [phase1])
        return wrap(phase2)
    except Exception as e:
        Log.error("problem replacing locals in\n{{phase1}}", phase1=phase1, cause=e)
873578
function makeRelPath(sourceUrl, targetUrl) {
  var source = sourceUrl.pathname;
  var target = targetUrl.pathname;
  var s = source.split('/');
  var t = target.split('/');
  while (s.length && s[0] === t[0]){
    s.shift();
    t.shift();
  }
  for (var i = 0, l = s.length - 1; i < l; i++) {
    t.unshift('..');
  }
  // empty '#' is discarded but we need to preserve it.
  var hash = (targetUrl.href.slice(-1) === URL_HASH) ? URL_HASH : targetUrl.hash;
  return t.join('/') + targetUrl.search + hash;
}
573506
function url(path) {
        if (path) {
            if (_.startsWith(path, "/")) {
                if (!_urlPrefix) {
                    // lets discover the base url via the base html element
                    _urlPrefix = $('base').attr('href') || "";
                    if (_.endsWith(_urlPrefix, '/')) {
                        _urlPrefix = _urlPrefix.substring(0, _urlPrefix.length - 1);
                    }
                }
                if (_urlPrefix) {
                    return _urlPrefix + path;
                }
            }
        }
        return path;
    }
837921
def make_url_absolute(self, url, resolve_base=False):
        

        if self.config['url']:
            if resolve_base:
                ubody = self.doc.unicode_body()
                base_url = find_base_url(ubody)
                if base_url:
                    return urljoin(base_url, url)
            return urljoin(self.config['url'], url)
        else:
            return url
759999
def makeMNBaseURL(url):
    """"""
    o = urllib.parse.urlparse(url, scheme=d1_common.const.DEFAULT_MN_PROTOCOL)
    if o.netloc and o.path:
        netloc = o.netloc
        path = o.path
    elif o.netloc:
        netloc = o.netloc
        path = d1_common.const.DEFAULT_MN_PATH
    elif o.path:
        s = o.path.split('/', 1)
        netloc = s[0]
        if len(s) == 1:
            path = d1_common.const.DEFAULT_MN_PATH
        else:
            path = s[1]
    else:
        netloc = d1_common.const.DEFAULT_MN_HOST
        path = d1_common.const.DEFAULT_MN_PATH
    return urllib.parse.urlunparse(
        (o.scheme, netloc, path, o.params, o.query, o.fragment)
    )
660637
def _build_url(*args, **kwargs) -> str:
    
    resource_url = API_RESOURCES_URLS
    for key in args:
        resource_url = resource_url[key]

    if kwargs:
        resource_url = resource_url.format(**kwargs)

    return urljoin(URL, resource_url)
1042809
def extracthtmlurls(mesg):
    
    chunk = HTMLChunker()
    chunk.feed(mesg)
    chunk.close()
    # above_context = 1
    # below_context = 1

    def somechunkisurl(chunks):
        for chnk in chunks:
            if chnk.url is not None:
                return True
        return False

    return extract_with_context(chunk.rval, somechunkisurl, 1, 1)
652102
def _grabContentFromUrl(self, url):
        
        # Defining an empty object for the response
        response = {}
                
        # This part has to be modified...        
        try:            
            # Configuring the socket
            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, self.info["host"], int(self.info["port"]), True)
            s = socks.socksocket()
            
            # Extracting the domain from the URL
            domain = self.getDomainFromUrl(url)        
            s.connect((domain, 80))

            message = 'GET ' + url + ' HTTP/1.0\r\n\r\n'
            s.sendall(message)
            
            data = ""
            while True:
                reply = s.recv(4096)

                if not reply:
                    break    
                else:
                    data += reply    
            
            # Processing data as expected
            response = self._createDataStructure(data)     
            
        # Try to make the errors clear for other users
        except socks.ProxyConnectionError, sPCE:
            errMsg = "ERROR socks.ProxyConnectionError. Something seems to be wrong with the Tor Bundler."   
            raise Exception( errMsg + " " + str(sPCE))        
            
        return response
979249
def urljoin(base, path=None):
    """"""
    # /foo/bar + baz makes /foo/bar/baz instead of /foo/baz
    if path is None:
        url = base
    else:
        if not base.endswith('/'):
            base += '/'
        url = urllib.parse.urljoin(base, str(path))
    return url
1323344
private URL extractActualUrl(URL jarUrl) throws MalformedURLException {
        String urlFile = jarUrl.getFile();
        int separatorIndex = urlFile.indexOf("!/");
        if (separatorIndex != -1) {
            String jarFile = urlFile.substring(0, separatorIndex);

            try {
                return new URL(jarFile);
            } catch (MalformedURLException var5) {
                if (!jarFile.startsWith("/")) {
                    jarFile = "/" + jarFile;
                }

                return new URL("file:" + jarFile);
            }
        } else {
            return jarUrl;
        }
    }
437806
def urljoin (parent, url):
    
    if urlutil.url_is_absolute(url):
        return url
    return urlparse.urljoin(parent, url)
193935
function (url) {
        url = _.trimStart(url);
        var pathVariables,
            p = {
                raw: url
            };

        // extract the protocol
        p.protocol = _.get(url.match(regexes.extractProtocol), GET_1);
        _.isString(p.protocol) && (url = url.substr(p.protocol.length + 3)); // remove that damn protocol from url

        // extract the host
        p.host = _.get(url.match(regexes.extractHost), GET_1);

        // if host exists there are a lot you can extract from it
        if (_.isString(p.host)) {
            url = url.substr(p.host.length); // remove host from url

            if ((p.auth = _.get(p.host.match(regexes.extractHostAuth), GET_1))) {
                p.host = p.host.substr(p.auth.length + 1); // remove auth from host
                p.auth = p.auth.split(AUTH_SEPARATOR);
                p.auth = {
                    user: p.auth[0],
                    password: p.auth[1]
                };
            }

            // extract the port from the host
            p.port = _.get(p.host.match(regexes.extractPort), GET_1);
            p.port && (p.host = p.host.substring(0, p.host.length - p.port.length - 1)); // remove port from url

            p.host = _.trim(p.host, DOMAIN_SEPARATOR).split(regexes.splitDomain); // split host by subdomains
        }

        // extract the path
        p.path = _.get(url.match(regexes.extractPath), GET_0);
        if (_.isString(p.path)) {
            url = url.substr(p.path.length);
            p.path && (p.path = p.path.replace(regexes.trimPath, MATCH_1)); // remove leading slash for valid path
            // if path is blank string, we set it to undefined, if '/' then single blank string array
            p.path = p.path ? (p.path === PATH_SEPARATOR ? [E] : p.path.split(PATH_SEPARATOR)) : undefined;
        }

        // extract the query string
        p.query = _.get(url.match(regexes.extractQuery), GET_1);
        _.isString(p.query) && ((url = url.substr(p.query.length + 1)), (p.query = QueryParam.parse(p.query)));

        // extract the hash
        p.hash = _.get(url.match(regexes.extractSearch), GET_1);

        // extract path variables
        pathVariables = _.transform(p.path, function (res, segment) {
            // check if the segment has path variable prefix followed by the variable name.
            if (_.startsWith(segment, PATH_VARIABLE_IDENTIFIER) && segment !== PATH_VARIABLE_IDENTIFIER) {
                res.push({ key: segment.slice(1) }); // remove path variable prefix.
            }
        }, []);
        p.variable = pathVariables.length ? pathVariables : undefined;

        return p;
    }
1020234
def build_url (self):
        
        # norm base url - can raise UnicodeError from url.idna_encode()
        base_url, is_idn = url_norm(self.base_url, self.encoding)
        # make url absolute
        if self.base_ref:
            # use base reference as parent url
            if ":" not in self.base_ref:
                # some websites have a relative base reference
                self.base_ref = urljoin(self.parent_url, self.base_ref)
            self.url = urljoin(self.base_ref, base_url)
        elif self.parent_url:
            # strip the parent url query and anchor
            urlparts = list(urlparse.urlsplit(self.parent_url))
            urlparts[4] = ""
            parent_url = urlutil.urlunsplit(urlparts)
            self.url = urljoin(parent_url, base_url)
        else:
            self.url = base_url
        # urljoin can unnorm the url path, so norm it again
        urlparts = list(urlparse.urlsplit(self.url))
        if urlparts[2]:
            urlparts[2] = urlutil.collapse_segments(urlparts[2])
        self.url = urlutil.urlunsplit(urlparts)
        # split into (modifiable) list
        self.urlparts = strformat.url_unicode_split(self.url)
        self.build_url_parts()
        # and unsplit again
        self.url = urlutil.urlunsplit(self.urlparts)
1023336
def _extract_links(self, selector, response_url, response_encoding, base_url):
        '''
        
        '''
        links = []
        # hacky way to get the underlying lxml parsed document
        for el, attr, attr_val in self._iter_links(selector.root):
            # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
            try:
                attr_val = urljoin(base_url, attr_val)
            except ValueError:
                continue # skipping bogus links
            else:
                url = self.process_attr(attr_val)
                if url is None:
                    continue
            # added 'ignore' to encoding errors
            url = to_native_str(url, encoding=response_encoding,
                                errors='ignore')
            # to fix relative links after process_value
            url = urljoin(response_url, url)
            link = Link(url, _collect_string_content(el) or u'',
                        nofollow=rel_has_nofollow(el.get('rel')))
            links.append(link)
        return self._deduplicate_if_needed(links)
291635
def complete_url(self, url):
    
    if self.base_url:
      return urlparse.urljoin(self.base_url, url)
    else:
      return url
633207
def _make_path(self, items):
        '''
        '''
        for key, val in items.items():
            if not isinstance(val, basestring):
                raise TypeError, ('Value "%s" for key "%s" must be an instance'
                                  ' of basestring' % (val, key))
            items[key] = quote_plus(val)

        try:
            path = self._url_format.format(**items)
        except AttributeError:
            # Old version of python
            path = self._url_format
            for key, val in items.items():
                path = path.replace('{%s}' % key, val)
        return path