Ask robots.txt allowance.
961359
def allows_url (self, url_data):
        """"""
        roboturl = url_data.get_robots_txt_url()
        with self.get_lock(roboturl):
            return self._allows_url(url_data, roboturl)
737056
def do_GET(self):
        """"""
        self.do_HEAD()
        template = pkg_resources.resource_string(__name__, "static/splash.html")
        page = template.decode('utf-8').format(self.headers.get("User-Agent"), self.cache)
        self.wfile.write(page.encode('utf-8'))
431107
def rnumlistwithreplacement(howmany, max, min=0):
    """"""
    if checkquota() < 1:
        raise Exception("Your www.random.org quota has already run out.")
    requestparam = build_request_parameterWR(howmany, min, max)
    request = urllib.request.Request(requestparam)
    request.add_header('User-Agent', 'randomwrapy/0.1 very alpha')
    opener = urllib.request.build_opener()
    numlist = opener.open(request).read()
    return numlist.split()
102662
def allowed(self, url, agent):
        ''''''
        return self.get(url).allowed(url, agent)
1171048
def _start(self):
        '''

        '''
        if self.whoami is None:
            me = self.get_me()
            if me.get('ok', False):
                self.whoami = me['result']
            else:
                raise ValueError('Bot Cannot request information, check '
                                 'api_key')
1077066
def provide_data(self):
        """"""
        return [(k, tuple([self.batch_size] + list(v.shape[1:]))) for k, v in self.data]
1187568
def robots(request):
    """"""

    resp = request.response
    resp.status = '200 OK'
    resp.content_type = 'text/plain'
    resp.body = """
User-Agent: *
Disallow: /
"""
    return resp
38291
def fetch(self, url):
        ''''''
        robots = Robots.fetch(
            url, ttl_policy=self.ttl_policy, *self.args, **self.kwargs)
        return (robots.expires, robots.agent(self.agent))
684966
function showUsage(req, res) {
	var egPayload = {
		"username": "soandso",
		"message": "Hello bot",
		"vars": {
			"name": "Soandso"
		}
	};
	res.writeHead(200, {"Content-Type": "text/plain"});
	res.write("Usage: curl -i \\\n");
	res.write("   -H \"Content-Type: application/json\" \\\n");
	res.write("   -X POST -d '" + JSON.stringify(egPayload) + "' \\\n");
	res.write("   http://localhost:2001/reply");
	res.end();
}
1228614
def get_requirements():
    """"""
    with open(os.path.join(topdir, 'requirements.txt')) as fin:
        lines = fin.readlines()
    lines = [line.strip() for line in lines]
    return lines
369619
def get(self, url):
        ''''''
        robots_url = Robots.robots_url(url)
        if robots_url not in self.cache:
            self.cache[robots_url] = ExpiringObject(partial(self.factory, robots_url))
        return self.cache[robots_url].get()
1282417
def request_search(self, txt=None):
        
        if self.checkBoxRegex.isChecked():
            try:
                re.compile(self.lineEditSearch.text(), re.DOTALL)
            except sre_constants.error as e:
                self._show_error(e)
                return
            else:
                self._show_error(None)

        if txt is None or isinstance(txt, int):
            txt = self.lineEditSearch.text()
        if txt:
            self.job_runner.request_job(
                self._exec_search, txt, self._search_flags())
        else:
            self.job_runner.cancel_requests()
            self._clear_occurrences()
            self._on_search_finished()
151200
def herokuapp_robots_view(request):
    

    response_text = ""
    http_host = request.get_host()

    if http_host and 'herokuapp' in http_host:
        response_text = "User-agent: *\nDisallow: /"

    return HttpResponse(response_text, content_type="text/plain")
1281032
def ask(message, options):
    """"""
    while 1:
        if os.environ.get('PIP_NO_INPUT'):
            raise Exception('No input was expected ($PIP_NO_INPUT set); question: %s' % message)
        response = raw_input(message)
        response = response.strip().lower()
        if response not in options:
            print('Your response (%r) was not one of the expected responses: %s' % (
                response, ', '.join(options)))
        else:
            return response
56298
def request_data(self):
        ''''''
        def on_tasks_received(info, error=None):
            if error or not info:
                logger.error('BTBrowserDialog.on_tasks_received: %s, %s.' %
                             (info, error))
                return
            if 'magnet_info' in info:
                tasks = info['magnet_info']
            elif 'torrent_info' in info:
                tasks = info['torrent_info']['file_info']
                self.file_sha1 = info['torrent_info']['sha1']
            elif 'error_code' in info:
                logger.error('BTBrowserDialog.on_tasks_received: %s, %s.' %
                             (info, error))
                self.app.toast(info.get('error_msg', ''))
                return
            else:
                logger.error('BTBrowserDialog.on_tasks_received: %s, %s.' %
                             (info, error))
                self.app.toast(_('Unknown error occured: %s') % info)
                return
            for task in tasks:
                size = int(task['size'])
                human_size = util.get_human_size(size)[0]
                select = (size > MIN_SIZE_TO_CHECK or 
                          task['file_name'].endswith(CHECK_EXT))
                self.liststore.append([
                    select,
                    task['file_name'],
                    size,
                    human_size,
                ])

        if self.source_url.startswith('magnet'):
            gutil.async_call(pcs.cloud_query_magnetinfo, self.app.cookie,
                             self.app.tokens, self.source_url, self.save_path,
                             callback=on_tasks_received)
        else:
            gutil.async_call(pcs.cloud_query_sinfo, self.app.cookie,
                             self.app.tokens, self.source_url,
                             callback=on_tasks_received)
747690
def _allows_url (self, url_data, roboturl):
        """"""
        with cache_lock:
            if roboturl in self.cache:
                self.hits += 1
                rp = self.cache[roboturl]
                return rp.can_fetch(self.useragent, url_data.url)
            self.misses += 1
        kwargs = dict(auth=url_data.auth, session=url_data.session)
        if hasattr(url_data, "proxy") and hasattr(url_data, "proxy_type"):
            kwargs["proxies"] = {url_data.proxytype: url_data.proxy}
        rp = robotparser2.RobotFileParser(**kwargs)
        rp.set_url(roboturl)
        rp.read()
        with cache_lock:
            self.cache[roboturl] = rp
        self.add_sitemap_urls(rp, url_data, roboturl)
        return rp.can_fetch(self.useragent, url_data.url)
915885
def _should_fetch_reason_with_robots(self, request: Request) -> Tuple[bool, str]:
        '''
        '''
        result = yield from \
            self._fetch_rule.check_initial_web_request(self._item_session, request)
        return result