Rss Crawler are supported if by every site containing an rss feed.

        Determines if this crawler works on the given url.

        :param str url: The url to test
        :return bool: Determines wether this crawler work on the given url
337809
def supports_site(url):
        

        # Follow redirects
        opener = urllib2.build_opener(urllib2.HTTPRedirectHandler)
        redirect = opener.open(url).url
        response = urllib2.urlopen(redirect).read()

        # Check if a standard rss feed exists
        return re.search(re_rss, response.decode('utf-8')) is not None
1169863
def parse(self, url):
        
        parsed_url = urlparse.urlparse(url)
        try:
            default_config = self.CONFIG[parsed_url.scheme]
        except KeyError:
            raise ValueError(
                'unrecognised URL scheme for {}: {}'.format(
                    self.__class__.__name__, url))
        handler = self.get_handler_for_scheme(parsed_url.scheme)
        config = copy.deepcopy(default_config)
        return handler(parsed_url, config)
918844
def fetch_url(url):
    

    with closing(urllib.urlopen(url)) as f:
        if f.code is 200:
            response = f.read()
            return strip_formfeeds(response).decode(ENCODING)
1107594
def sitemap_check(url):
        
        response = urllib2.urlopen(UrlExtractor.get_sitemap_url(url, True))

        # Check if "Sitemap" is set
        return "Sitemap:" in response.read().decode('utf-8')
725561
def url_should_contain(self, url):
    """"""

    if url not in world.browser.current_url:
        raise AssertionError(
            "Browser URL expected to contain {!r}, got {!r}.".format(
                url, world.browser.current_url))
558589
def is_external_url(self, url, site_url):
        
        url_splitted = urlsplit(url)
        if not url_splitted.netloc:
            return False
        return url_splitted.netloc != urlsplit(site_url).netloc
483849
def visit(self, url=''):
        
        def _visit(url):
            if len(url) > 0 and url[0] == '/':
                # url's first character is a forward slash; treat as relative path
                path = url
                full_url = self.driver.current_url
                parsed_url = urlparse(full_url)
                base_url = str(parsed_url.scheme) + '://' + str(parsed_url.netloc)
                url = urljoin(base_url, path)

            try:
                return self.driver.get(url)
            except TimeoutException:
                if self.ignore_page_load_timeouts:
                    pass
                else:
                    raise PageTimeoutException.PageTimeoutException(self, url)

        return self.execute_and_handle_webdriver_exceptions(lambda: _visit(url))
442872
def item_enclosure_url(self, item):
        
        try:
            url = item.image.url
        except (AttributeError, ValueError):
            img = BeautifulSoup(item.html_content, 'html.parser').find('img')
            url = img.get('src') if img else None
        self.cached_enclosure_url = url
        if url:
            url = urljoin(self.site_url, url)
            if self.feed_format == 'rss':
                url = url.replace('https://', 'http://')
        return url
823073
def match(self, url):
        '''
        
        '''
        try:
            urlSchemes = self._urlSchemes.itervalues() # Python 2
        except AttributeError:
            urlSchemes = self._urlSchemes.values() # Python 3

        for urlScheme in urlSchemes:
            if urlScheme.match(url):
                return True
        return False
291635
def complete_url(self, url):
    
    if self.base_url:
      return urlparse.urljoin(self.base_url, url)
    else:
      return url
1106691
def content_allows_robots (self):
        
        if not self.is_html():
            return True
        # construct parser object
        handler = linkparse.MetaRobotsFinder()
        parser = htmlsax.parser(handler)
        handler.parser = parser
        if self.charset:
            parser.encoding = self.charset
        # parse
        try:
            parser.feed(self.get_content())
            parser.flush()
        except linkparse.StopParse as msg:
            log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
            pass
        # break cyclic dependencies
        handler.parser = None
        parser.handler = None
        return handler.follow
349007
def valid_url(url):
    
    regex = re.compile(
        r'^(?:http)s?://'
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+'
        r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?))'
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
    if not regex.match(url):
        raise argparse.ArgumentTypeError('"{0}" is invalid url.'.format(url))
    return url
590661
def is_valid_url(value):
    
    match = URL_REGEX.match(value)
    host_str = urlparse(value).hostname
    return match and is_valid_host(host_str)
1103150
public static boolean containsURL(final URL[] urls, final URL url) {
        checkNotNull("urls", urls);
        checkNotNull("url", url);
        for (int i = 0; i < urls.length; i++) {
            final URL element = urls[i];
            final String elementStr = element.toExternalForm();
            final String urlStr = url.toExternalForm();
            if (elementStr.equals(urlStr)) {
                return true;
            }
        }
        return false;
    }
765434
public final static boolean isValidURL(String url) {
        if (url == null) {
            return false;
        } else {
            return Patterns.WEB_URL.matcher(url).matches();
        }
    }
390223
def is_safe_url(url, host=None):
    
    if url is not None:
        url = url.strip()
    if not url:
        return False
    if six.PY2:  # pragma: nocover
        try:
            url = force_text(url)
        except UnicodeDecodeError:
            return False
    # Chrome treats \ completely as / in paths but it could be part of some
    # basic auth credentials so we need to check both URLs.
    return _is_safe_url(url, host) and _is_safe_url(url.replace('\\', '/'), host)
420318
def check(self, url: str) -> Optional[dict]:
        
        data = self.data.get(url)
        if data:
            data = self._check_expiration(url, data)
        return data.data if data else None