Return whether we cannot follow links due to robots.txt directives.
330832
def robots_cannot_follow(cls, element):
        '''
        '''
        return (
            element.tag == 'meta'
            and element.attrib.get('name', '').lower() == 'robots'
            and 'nofollow' in element.attrib.get('value', '').lower()
        )
300710
def should_see_link_text(self, link_text, link_url):
    """"""

    elements = ElementSelector(
        world.browser,
        str('//a[@href="%s"][./text()="%s"]' % (link_url, link_text)),
        filter_displayed=True,
    )
    if not elements:
        raise AssertionError("Expected link not found.")
931747
def _is_skippable(filename_full):
    """"""

    # File types
    if not Settings.follow_symlinks and os.path.islink(filename_full):
        return True
    if os.path.basename(filename_full) == timestamp.RECORD_FILENAME:
        return True
    if not os.path.exists(filename_full):
        if Settings.verbose:
            print(filename_full, 'was not found.')
        return True

    return False
766376
function checkBrokenLinks(docs, apis, options) {
  var byFullId = Object.create(null);

  docs.forEach(function(doc) {
    byFullId[doc.section + '/' + doc.id] = doc;
    if (apis[doc.section]) {
      doc.anchors.push('directive', 'service', 'filter', 'function');
    }
  });

  docs.forEach(function(doc) {
    doc.links.forEach(function(link) {
      if (options && !options.html5Mode) {
        link = link.substring(2);
      }
      // convert #id to path#id
      if (link[0] == '#') {
        link = doc.section + '/' + doc.id.split('#').shift() + link;
      }

      var parts = link.split('#');
      var pageLink = parts[0];
      var anchorLink = parts[1];
      var linkedPage = byFullId[pageLink];

      if (!linkedPage) {
        console.log('WARNING: ' + doc.section + '/' + doc.id + ' (defined in ' + doc.file + ') points to a non existing page "' + link + '"!');
      } else if (anchorLink && linkedPage.anchors.indexOf(anchorLink) === -1) {
        console.log('WARNING: ' + doc.section + '/' + doc.id + ' (defined in ' + doc.file + ') points to a non existing anchor "' + link + '"!');
      }
    });
  });
}
551187
def _accept_as_blank(self, url_info: URLInfo):
        ''''''
        _logger.debug(__('Got empty robots.txt for {0}.', url_info.url))
        self._robots_txt_pool.load_robots_txt(url_info, '')
641907
def isvalid(self):
        

        gcontents = [gf.rstrip('\n') for gf in self.repo.bake('ls-files')()]
        fcontents = os.listdir(self.repopath)
        return all([sf in gcontents for sf in std_files]) and all([sf in fcontents for sf in std_files])
432300
def isignored(self, relpath, directory=False):
    """"""
    relpath = self._relpath_no_dot(relpath)
    if directory:
      relpath = self._append_trailing_slash(relpath)
    return self.ignore.match_file(relpath)
415147
def ensure_compliance(self):
        
        for p in self.paths:
            if os.path.exists(p):
                if self.is_compliant(p):
                    continue

                log('File %s is not in compliance.' % p, level=INFO)
            else:
                if not self.always_comply:
                    log("Non-existent path '%s' - skipping compliance check"
                        % (p), level=INFO)
                    continue

            if self._take_action():
                log("Applying compliance criteria to '%s'" % (p), level=INFO)
                self.comply(p)
343915
private boolean isSymlink(File base, Vector pathElements) {
        if (pathElements.size() > 0) {
            String current = (String) pathElements.remove(0);
            try {
                return FileUtil.isSymbolicLink(base, current)
                    || isSymlink(new File(base, current), pathElements);
            } catch (IOException ioe) {
                String msg = "IOException caught while checking "
                    + "for links, couldn't get canonical path!";
                // will be caught and redirected to Ant's logging system
                System.err.println(msg);
            }
        }
        return false;
    }
854616
protected function checkBrowserYandexBlogsBot()
    {
        if (stripos($this->_agent, 'YandexBlogs') !== false) {
            $aresult = explode('/', stristr($this->_agent, 'YandexBlogs'));
            if (isset($aresult[1])) {
                $aversion = explode(' ', $aresult[1]);
                $this->setVersion(str_replace(';', '', $aversion[0]));
                $this->_browser_name = self::BROWSER_YANDEXBLOGS_BOT;
                $this->setRobot(true);
                return true;
            }
        }
        return false;
    }
920124
def is_file_ignored(opts, fname):
    '''
    
    '''
    if opts['file_ignore_regex']:
        for regex in opts['file_ignore_regex']:
            if re.search(regex, fname):
                log.debug(
                    'File matching file_ignore_regex. Skipping: %s',
                    fname
                )
                return True

    if opts['file_ignore_glob']:
        for glob in opts['file_ignore_glob']:
            if fnmatch.fnmatch(fname, glob):
                log.debug(
                    'File matching file_ignore_glob. Skipping: %s',
                    fname
                )
                return True
    return False
790697
def get_robotstxt_parser(url, session=None):
    """"""
    rp = robotparser.RobotFileParser()
    try:
        req = urlopen(url, session, max_content_bytes=MaxContentBytes, raise_for_status=False)
    except Exception:
        # connect or timeout errors are treated as an absent robots.txt
        rp.allow_all = True
    else:
        if req.status_code >= 400:
            rp.allow_all = True
        elif req.status_code == 200:
            rp.parse(req.text.splitlines())
    return rp
501245
def _process_robots(self):
        '''
        '''
        try:
            self._item_session.request = request = self._new_initial_request(with_body=False)
            verdict, reason = (yield from self._should_fetch_reason_with_robots(
                request))
        except REMOTE_ERRORS as error:
            _logger.error(
                _('Fetching robots.txt for ‘{url}’ '
                  'encountered an error: {error}'),
                url=self._next_url_info.url, error=error
            )
            self._result_rule.handle_error(self._item_session, error)

            wait_time = self._result_rule.get_wait_time(
                self._item_session, error=error
            )

            if wait_time:
                _logger.debug('Sleeping {0}.', wait_time)
                yield from asyncio.sleep(wait_time)

            return False
        else:
            _logger.debug('Robots filter verdict {} reason {}', verdict, reason)

            if not verdict:
                self._item_session.skip()
                return False

        return True
1356928
def validate_links(links):
	
	valid_links = []
	for link in links:
		if link[:7] in "http://" or link[:8] in "https://":
			valid_links.append(link)
	
	if not valid_links:
		print("No files found.")
		sys.exit(0)

	# checking valid urls for return code
	urls = {}
	for link in valid_links:
		if 'github.com' and '/blob/' in link:
			link = link.replace('/blob/', '/raw/')
		urls[link] = {'code': get_url_nofollow(link)}
		
	
	# printing valid urls with return code 200
	available_urls = []
	for url in urls:
		print("code: %d\turl: %s" % (urls[url]['code'], url))
		if urls[url]['code'] != 0:
			available_urls.append(url)

	return available_urls
864874
def notblocked(page):
    """"""
    for blocked in PAGES_TO_BLOCK:
        if blocked[0] != '*':
            blocked = '*' + blocked
        rx = re.compile(blocked.replace('*', '[^$]*'))
        if rx.match(page):
            return False
    return True
961359
def allows_url (self, url_data):
        """"""
        roboturl = url_data.get_robots_txt_url()
        with self.get_lock(roboturl):
            return self._allows_url(url_data, roboturl)
1201720
def robots(self):
        """"""
        r = 'noindex' if self.is_noindex else 'index'
        r += ','
        r += 'nofollow' if self.is_nofollow else 'follow'
        return r