Consult by fetching robots.txt as needed.

        Args:
            request: The request to be made
                to get the file.

        Returns:
            True if can fetch

        Coroutine
543290
def consult_robots_txt(self, request: HTTPRequest) -> bool:
        '''
        '''
        if not self._robots_txt_checker:
            return True

        result = yield from self._robots_txt_checker.can_fetch(request)
        return result
228318
async def is_try_or_pull_request(self):
        
        tasks = [asyncio.ensure_future(link.is_try_or_pull_request()) for link in self.links]
        tasks.insert(0, asyncio.ensure_future(is_try_or_pull_request(self.context, self.task)))

        conditions = await raise_future_exceptions(tasks)
        return any(conditions)
818652
def get(self, request, uri):
        
        uri = self.decode_uri(uri)
        node = cio.load(uri)
        return self.render_to_json(node)
267465
def matches(self, request):
        
        return (
            partial_path_match(request.path_info, self.path) and
            request.method == self.method
        )
984452
def runSearchReads(self, request):
        
        return self.runSearchRequest(
            request, protocol.SearchReadsRequest,
            protocol.SearchReadsResponse,
            self.readsGenerator)
905525
def fetch(self):
        
        raw = yield gen.Task(self._redis_client.get, self._key)
        if raw:
            self.loads(base64.b64decode(raw))
            raise gen.Return(True)
        raise gen.Return(False)
1219372
def pull_request(self, file):
        
        uri = "{api}/repos/{upstream}/pulls".format(
            api=self.github_api_url,
            upstream=self.upstream,
            path=file.path
        )
        params = {
          "title": "[Proxy] {message}".format(message=file.logs),
          "body": "",
          "head": "{origin}:{branch}".format(origin=self.origin.split("/")[0], branch=file.branch),
          "base": self.master_upstream
        }
        data = self.request("POST", uri, data=params)

        if data.status_code == 201:
            return json.loads(data.content.decode("utf-8"))["html_url"]
        else:
            reply = json.loads(data.content.decode("utf-8"))
            return self.ProxyError(
                data.status_code, reply["message"],
                step="pull_request", context={
                    "uri": uri,
                    "params": params
                }
            )
1346908
def read_list(self, request):
        
        pipeline = crud_pipeline_factory.read_list_pipeline(
            configuration=self.configuration)
        return pipeline(request=request)
278551
def _fetch(self, uri, sub_domain=False):
        
        url = self._build_request_url(uri, sub_domain=sub_domain)

        try:
            res = self.request_handler.send("HEAD", url=url, allow_redirects=self.follow_redirects)
            if res.status_code not in self.ignored_error_codes:
                self._log_response(res.status_code, url, res.headers)
        except (AttributeError, RequestHandlerException):
            # res is None or another error occurred
            pass
844019
def _prepare_request_file_vs_dir(self, request: Request) -> bool:
        '''
        '''
        if self._item_session.url_record.link_type:
            is_file = self._item_session.url_record.link_type == LinkType.file
        elif request.url_info.path.endswith('/'):
            is_file = False
        else:
            is_file = 'unknown'

        if is_file == 'unknown':
            files = yield from self._fetch_parent_path(request)

            if not files:
                return True

            filename = posixpath.basename(request.file_path)

            for file_entry in files:
                if file_entry.name == filename:
                    _logger.debug('Found entry in parent. Type {}',
                                  file_entry.type)
                    is_file = file_entry.type != 'dir'
                    break
            else:
                _logger.debug('Did not find entry. Assume file.')
                return True

            if not is_file:
                request.url = append_slash_to_path_url(request.url_info)
                _logger.debug('Request URL changed to {}. Path={}.',
                              request.url, request.file_path)

        return is_file
135302
def _doRequest(self, request=None, is_file=False, file_xml_uri=''):
		""""""
		if request is None:
			request = []

		if is_file and file_xml_uri:
			url = self._buildFileUrl(file_xml_uri)
		else:
			request = '&'.join(request)
			url = "%s?%s" % (self._buildUrl(), request)

		if self._debug:
			print '[PyFileMaker DEBUG] ', url

		resp = requests.get(
			url = url,
			auth = (self._login, self._password)
		)
		resp.raise_for_status()

		return resp.content
1131601
def get(self, request):
        
        headers = {"Authorization": "Bearer " + self.token }
        g = requests.get(self.url + request, headers=headers)
        return json.loads(g.text)
101503
function handleTernGetFile(request) {

            function replyWith(name, txt) {
                _postMessageByPass({
                    type: MessageIds.TERN_GET_FILE_MSG,
                    file: name,
                    text: txt
                });
            }

            var name = request.file;

            /**
             * Helper function to get the text of a given document and send it to tern.
             * If DocumentManager successfully gets the file's text then we'll send it to the tern node domain.
             * The Promise for getDocumentText() is returned so that custom fail functions can be used.
             *
             * @param {string} filePath - the path of the file to get the text of
             * @return {jQuery.Promise} - the Promise returned from DocumentMangaer.getDocumentText()
             */
            function getDocText(filePath) {
                if (!FileSystem.isAbsolutePath(filePath) || // don't handle URLs
                        filePath.slice(0, 2) === "//") { // don't handle protocol-relative URLs like //example.com/main.js (see #10566)
                    return (new $.Deferred()).reject().promise();
                }

                var file = FileSystem.getFileForPath(filePath),
                    promise = DocumentManager.getDocumentText(file);

                promise.done(function (docText) {
                    resolvedFiles[name] = filePath;
                    numResolvedFiles++;
                    replyWith(name, filterText(docText));
                });
                return promise;
            }

            /**
             * Helper function to find any files in the project that end with the
             * name we are looking for.  This is so we can find requirejs modules
             * when the baseUrl is unknown, or when the project root is not the same
             * as the script root (e.g. if you open the 'brackets' dir instead of 'brackets/src' dir).
             */
            function findNameInProject() {
                // check for any files in project that end with the right path.
                var fileName = name.substring(name.lastIndexOf("/") + 1);

                function _fileFilter(entry) {
                    return entry.name === fileName;
                }

                ProjectManager.getAllFiles(_fileFilter).done(function (files) {
                    var file;
                    files = files.filter(function (file) {
                        var pos = file.fullPath.length - name.length;
                        return pos === file.fullPath.lastIndexOf(name);
                    });

                    if (files.length === 1) {
                        file = files[0];
                    }
                    if (file) {
                        getDocText(file.fullPath).fail(function () {
                            replyWith(name, "");
                        });
                    } else {
                        replyWith(name, "");
                    }
                });
            }

            if (!isFileExcludedInternal(name)) {
                getDocText(name).fail(function () {
                    getDocText(rootTernDir + name).fail(function () {
                        // check relative to project root
                        getDocText(projectRoot + name)
                            // last look for any files that end with the right path
                            // in the project
                            .fail(findNameInProject);
                    });
                });
            }
        }
110443
def fetch_file(self, in_path, out_path):
        
        output = self.get_chain().call(
            ansible_mitogen.target.read_path,
            mitogen.utils.cast(in_path),
        )
        ansible_mitogen.target.write_path(out_path, output)
534472
def fetch_file(self, in_path, out_path):
        '''  '''
        vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host)
        try:
            self.sftp = self._connect_sftp()
        except:
            raise errors.AnsibleError("failed to open a SFTP connection")
        try:
            self.sftp.get(in_path, out_path)
        except IOError:
            raise errors.AnsibleError("failed to transfer file from %s" % in_path)
414752
def get(self, request, uri):
        
        uri = self.decode_uri(uri)
        node = cio.get(uri, lazy=False)

        if node.content is None:
            raise Http404

        return self.render_to_json({
            'uri': node.uri,
            'content': node.content
        })
1348389
def fetch_robots_txt(self, request: Request, file=None):
        '''
        '''
        url_info = request.url_info
        url = URLInfo.parse('{0}://{1}/robots.txt'.format(
            url_info.scheme, url_info.hostname_with_port)).url

        if not file:
            file = wpull.body.new_temp_file(os.getcwd(), hint='robots')

        with contextlib.closing(file):
            request = Request(url)

            session = self._web_client.session(request)
            while not session.done():
                wpull.util.truncate_file(file.name)

                try:
                    response = yield from session.start()
                    yield from session.download(file=file)
                except ProtocolError:
                    self._accept_as_blank(url_info)

                    return

            status_code = response.status_code

            if 500 <= status_code <= 599:
                raise ServerError('Server returned error for robots.txt.')

            if status_code == 200:
                self._read_content(response, url_info)
            else:
                self._accept_as_blank(url_info)