Return True iff we can recurse into the url's content.
557609
def allows_recursion (self):
        
        log.debug(LOG_CHECK, "checking recursion of %r ...", self.url)
        if not self.valid:
            log.debug(LOG_CHECK, "... no, invalid.")
            return False
        if not self.can_get_content():
            log.debug(LOG_CHECK, "... no, cannot get content.")
            return False
        if not self.allows_simple_recursion():
            return False
        if self.size > self.aggregate.config["maxfilesizeparse"]:
            log.debug(LOG_CHECK, "... no, maximum parse size.")
            return False
        if not self.is_parseable():
            log.debug(LOG_CHECK, "... no, not parseable.")
            return False
        if not self.content_allows_robots():
            log.debug(LOG_CHECK, "... no, robots.")
            return False
        log.debug(LOG_CHECK, "... yes, recursion.")
        return True
970507
protected boolean isRarArchive(URL url)
   {
      if (url == null)
         return false;

      return isRarFile(url) || isRarDirectory(url);
   }
1014491
def check_url(url_data, logger):
    """"""
    if url_data.has_result:
        logger.log_url(url_data.to_wire())
    else:
        cache = url_data.aggregate.result_cache
        key = url_data.cache_url
        result = cache.get_result(key)
        if result is None:
            # check
            check_start = time.time()
            try:
                url_data.check()
                do_parse = url_data.check_content()
                url_data.checktime = time.time() - check_start
                # Add result to cache
                result = url_data.to_wire()
                cache.add_result(key, result)
                for alias in url_data.aliases:
                    # redirect aliases
                    cache.add_result(alias, result)
                # parse content recursively
                # XXX this could add new warnings which should be cached.
                if do_parse:
                    parser.parse_url(url_data)
            finally:
                # close/release possible open connection
                url_data.close_connection()
        else:
            # copy data from cache and adjust it
            result = copy.copy(result)
            result.parent_url = url_data.parent_url
            result.base_ref = url_data.base_ref or u""
            result.base_url = url_data.base_url or u""
            result.line = url_data.line
            result.column = url_data.column
            result.level = url_data.recursion_level
            result.name = url_data.name
        logger.log_url(result)
1060698
def retrieve_content(self):
        """"""
        path = self._construct_path_to_source_content()
        res = self._http.get(path)
        self._populated_fields['content'] = res['content']
        return res['content']
246222
def _relIndexLink(self):
        """"""
        return os.path.join("..", os.path.basename(self.pathname), "index.html")
697828
def _get_content(self, url):
        ""
        target_url = self._db_url + '/' + unquote(url)    # .encode('utf-8'))
        log.debug("Opening '{0}'".format(target_url))
        try:
            f = self.opener.open(target_url)
        except HTTPError as e:
            log.error("HTTP error, your session may be expired.")
            log.error(e)
            if input("Request new permanent session and retry? (y/n)") in 'yY':
                self.request_permanent_session()
                return self._get_content(url)
            else:
                return None
        log.debug("Accessing '{0}'".format(target_url))
        try:
            content = f.read()
        except IncompleteRead as icread:
            log.critical(
                "Incomplete data received from the DB, " +
                "the data could be corrupted."
            )
            content = icread.partial
        log.debug("Got {0} bytes of data.".format(len(content)))
        return content.decode('utf-8')
449034
def geturl_req(url):
    """"""
    request = urllib.request.Request(url)
    request.add_header('Authorization', 'token %s' % API_TOKEN)
    try:
        response_url = urllib.request.urlopen(request).geturl()
        return response_url
    except urllib.error.HTTPError:
        exception()
        sys.exit(0)
685676
def is_url(value):
    ""
    if not isinstance(value, str):
        return False
    if not value.startswith('http://') and not value.startswith('https://'):
        return False
    # Any whitespace at all is invalid
    if whitespace_re.search(value):
        return False
    return True
1201649
public function isTranslable()
    {
        if ($this->getPath() === null) {
            $this->detectUrlDetails();
        }

        foreach ($this->config->getExcludedUrls() as $regex) {
            $escapedRegex = Text::escapeForRegex(Text::fullTrim($regex));
            $fullRegex = sprintf('/%s/', $escapedRegex);

            if (preg_match($fullRegex, $this->getPath()) === 1) {
                return false;
            }
        }

        return true;
    }
1147849
def guess_url(url):
    
    if url.lower().startswith("www."):
        # syntactic sugar
        return "http://%s" % url
    elif url.lower().startswith("ftp."):
        # syntactic sugar
        return "ftp://%s" % url
    return url
831188
function(url) {
  if (!allowedResources) {
    return true;
  }
  for (var i=0; i<allowedResources.length; i++) {
    var allow = allowedResources[i];
    var index = url.indexOf(allow);
    if (index == 0) {
      return true;
    }
  }
  return false;
}
1141136
def is_parsable(url):
        

        try:
            parsed = urlparse(url)
            URLHelper.__cache[url] = parsed
            return True
        except:
            return False
1215329
def is_safe_url(url, host=None):
    
    if not url:
        return False
    netloc = urlparse.urlparse(url)[1]
    return not netloc or netloc == host
715623
def safe_url(url):
    """"""
    parsed = urlparse(url)
    if parsed.password is not None:
        pwd = ':%s@' % parsed.password
        url = url.replace(pwd, ':*****@')
    return url
205456
def http_get(url, filename=None):
    """"""
    try:
        ret = requests.get(url)
    except requests.exceptions.SSLError as error:
        _LOGGER.error(error)
        return False

    if ret.status_code != 200:
        return False

    if filename is None:
        return ret.content

    with open(filename, 'wb') as data:
        data.write(ret.content)
    return True
482473
def _get_content_type(url):
        """"""
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
        if not scheme in ('http', 'https', 'ftp', 'ftps'):
            ## FIXME: some warning or something?
            ## assertion error?
            return ''
        req = Urllib2HeadRequest(url, headers={'Host': netloc})
        resp = urlopen(req)
        try:
            if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'):
                ## FIXME: doesn't handle redirects
                return ''
            return resp.info().get('content-type', '')
        finally:
            resp.close()
865771
def shouldSkipUrl(self, url, data):
        """"""
        return url in (
            self.stripUrl % '130217', # video
            self.stripUrl % '130218', # video
            self.stripUrl % '130226', # video
            self.stripUrl % '130424', # video
        )