Request URL and parse response. Yield a ``Torrent`` for every torrent
        on page.
541826
def items(self):
        
        request = get(str(self.url), headers={'User-Agent' : "Magic Browser","origin_req_host" : "thepiratebay.se"})
        root = html.fromstring(request.text)
        items = [self._build_torrent(row) for row in
                 self._get_torrent_rows(root)]
        for item in items:
            yield item
871980
def download_file(url):
    
    response = requests.get(url)
    if response.status_code is not 200:
        return None
    return response.text
267613
def __fetch_data(self, url):
        
        url += '&api_key=' + self.api_key
        try:
            response = urlopen(url)
            root = ET.fromstring(response.read())
        except HTTPError as exc:
            root = ET.fromstring(exc.read())
            raise ValueError(root.get('message'))
        return root
129147
def fetch_and_parse(method, uri, params_prefix=None, **params):
    """"""
    response = fetch(method, uri, params_prefix, **params)
    return _parse(json.loads(response.text))
431869
def parse(self, response):
        
        if not self.helper.parse_crawler.content_type(response):
            return

        for request in self.helper.parse_crawler \
                .recursive_requests(response, self, self.ignore_regex,
                                    self.ignore_file_extensions):
            yield request

        yield self.helper.parse_crawler.pass_to_pipeline_if_article(
            response, self.allowed_domains[0], self.original_url)
249674
def get(self, url, params=None, raw=False, stream=False, **request_kwargs):
        

        full_url = self.build_url(url)
        params = params or {}

        # Add token (if it's not already there)
        if self._token:
            params.setdefault('token', self._token)

        response = requests.get(full_url, params=params, stream=stream,
                                **request_kwargs)
        self.check_for_errors(response)  # Raise exception if something failed

        if stream:
            return response
        if raw or not response.content:
            return response.content
        return json.loads(response.text)
231438
def request(self, *args, **kwargs):
        
        response = requests.request(*args, **kwargs)
        return BeautifulSoup.BeautifulSoup(response.text, "html.parser")
124431
def _get_url(url):
    """"""
    try:
        data = HTTP_SESSION.get(url, stream=True)
        data.raise_for_status()
    except requests.exceptions.RequestException as exc:
        raise FetcherException(exc)

    return data
880170
def _iter_response(self, url, params=None):
        """"""
        if params is None:
            params = {}
        params['page_number'] = 1

        # Last page lists itself as next page
        while True:
            response = self._request(url, params)

            for item in response['result_data']:
                yield item

            # Last page lists itself as next page
            if response['service_meta']['next_page_number'] == params['page_number']:
                break

            params['page_number'] += 1
260815
def _fetch_one(self, request: Request) -> Tuple[bool, float]:
        '''
        '''
        _logger.info(_('Fetching ‘{url}’.'), url=request.url)

        response = None

        try:
            response = yield from self._web_client_session.start()
            self._item_session.response = response

            action = self._result_rule.handle_pre_response(self._item_session)

            if action in (Actions.RETRY, Actions.FINISH):
                raise HookPreResponseBreak()

            self._file_writer_session.process_response(response)

            if not response.body:
                response.body = Body(
                    directory=self._item_session.app_session.root_path,
                    hint='resp_cb'
                )

            yield from \
                self._web_client_session.download(
                    file=response.body,
                    duration_timeout=self._fetch_rule.duration_timeout
                )
        except HookPreResponseBreak:
            _logger.debug('Hook pre-response break.')
            return True, None
        except REMOTE_ERRORS as error:
            self._log_error(request, error)

            self._result_rule.handle_error(self._item_session, error)
            wait_time = self._result_rule.get_wait_time(
                self._item_session, error=error
            )

            if request.body:
                request.body.close()

            if response:
                response.body.close()

            return True, wait_time
        else:
            self._log_response(request, response)
            action = self._handle_response(request, response)
            wait_time = self._result_rule.get_wait_time(self._item_session)

            yield from self._run_coprocessors(request, response)

            response.body.close()

            if request.body:
                request.body.close()

            return action != Actions.NORMAL, wait_time
310086
def get_page(url):
    
    response = requests.get(url)
    handle_error(response, 200)
    data = response.json()
    return data
858624
def _get_request(url_root,api_key,path,response_type,params, ssl_verify):
    
    url = _url_builder(url_root,api_key,path,params)
    content = _fetch(url, ssl_verify)
    response = _dispatch(response_type)(content)
    return response
298103
def iterall(cls, connection=None, **kwargs):
        

        try:
            limit = kwargs['limit']
        except KeyError:
            limit = None

        try:
            page = kwargs['page']
        except KeyError:
            page = None

        def _all_responses():
            page = 1  # one based
            params = kwargs.copy()

            while True:
                params.update(page=page, limit=250)
                rsp = cls._make_request('GET', cls._get_all_path(), connection, params=params)
                if rsp:
                    yield rsp
                    page += 1
                else:
                    yield []  # needed for case where there is no objects
                    break

        if not (limit or page):
            for rsp in _all_responses():
                for obj in rsp:
                    yield cls._create_object(obj, connection=connection)

        else:
            response = cls._make_request('GET', cls._get_all_path(), connection, params=kwargs)
            for obj in cls._create_object(response, connection=connection):
                yield obj
483519
def _fetch(self, urls):
        
        responses = self._request_bulk(urls)
        for response in responses:
            try:
                soup = BeautifulSoup(response.content, 'html.parser',
                                     from_encoding="iso-8859-1")
                text = soup.get_text()
            except Exception:
                text = response.text
            self.data.append(text) # Opportunistic findings
        return responses
1118590
def _iterator(self, response):
        '''
        
        '''

        while True:
            result = deque(self._parse_data(response))
            del response
            if result:
                doc_count = len(result)
                last = result.pop()
                while result:
                    yield result.popleft()

                # We expect doc_count = self._page_size + 1 results, if
                # we have self._page_size or less it means we are on the
                # last page and need to return the last result.
                if doc_count < self._real_page_size:
                    yield last
                    break
                del result

                # if we are in a view, keys could be duplicate so we
                # need to start from the right docid
                if last['id']:
                    response = self._call(startkey=last['key'],
                                          startkey_docid=last['id'])
                # reduce result keys are unique by definition
                else:
                    response = self._call(startkey=last['key'])

            else:
                break
582609
def _items(self):
        
        torrents = map(self._get_torrent, self._get_rows())

        for t in torrents:
            yield t
1243363
def fetch(url, binary, outfile, noprint, rendered):
	'''
	
	'''
	with chrome_context.ChromeContext(binary=binary) as cr:
		resp = cr.blocking_navigate_and_get_source(url)
		if rendered:
			resp['content'] = cr.get_rendered_page_source()
			resp['binary'] = False
			resp['mimie'] = 'text/html'

	if not noprint:
		if resp['binary'] is False:
			print(resp['content'])
		else:
			print("Response is a binary file")
			print("Cannot print!")

	if outfile:
		with open(outfile, "wb") as fp:
			if resp['binary']:
				fp.write(resp['content'])
			else:
				fp.write(resp['content'].encode("UTF-8"))