Extracts the Rss Feed and initiates crawling it.

        :param obj response: The scrapy response
331892
def parse(self, response):
        
        yield scrapy.Request(self.helper.url_extractor.get_rss_url(response),
                             callback=self.rss_parse)
689451
def get_object(self):
        
        obj = super(PreviewWrapper, self).get_object()

        if not obj:
            raise http.Http404

        return obj
1004007
def get(request, obj_id=None):
    
    res = Result()
    if obj_id:
        if obj_id == '0':
            obj = {
                'id': 0,
                'name': 'TAGLESS',
                'artist': False,
            }
        else:
            obj = get_object_or_404(Tag, pk=obj_id).json()

        res.append(obj)
        return JsonResponse(res.asDict())
    else:
        if request.GET.get('count'):
            itags = Tag.objects.all().annotate(icount=Count('image'))
            vtags = Tag.objects.all().annotate(vcount=Count('video'))

            for i, tag in enumerate(itags):
                tag.count = itags[i].icount + vtags[i].vcount
                res.append(tag.json())
        else:
            for tag in Tag.objects.all():
                res.append(tag.json())

        return JsonResponse(res.asDict())
461412
function ScrapeError(msg, response, bodyString) {
  Error.call(this, msg);

  this.response = response;
  this.bodyString = bodyString;
}
134737
def _read_content(self, response: Response, original_url_info: URLInfo):
        ''''''
        data = response.body.read(4096)
        url_info = original_url_info

        try:
            self._robots_txt_pool.load_robots_txt(url_info, data)
        except ValueError:
            _logger.warning(__(
                _('Failed to parse {url} for robots exclusion rules. '
                  'Ignoring.'), url_info.url))
            self._accept_as_blank(url_info)
        else:
            _logger.debug(__('Got a good robots.txt for {0}.',
                             url_info.url))
413256
def scrape(self, url):
        '''
        
        '''
        if isinstance(url, str) is False:
            raise TypeError("The type of url must be str.")

        if self.readable_web_pdf is not None and self.readable_web_pdf.is_pdf_url(url) is True:
            web_data = self.readable_web_pdf.url_to_text(url)
        else:
            web_data = ""
            req = urllib.request.Request(url=url)
            with urllib.request.urlopen(req) as f:
                web = f.read().decode('utf-8')
                dom = pq(web)
                [dom(remove_object).remove() for remove_object in self.__remove_object_list]

                for dom_object in self.__dom_object_list:
                    web_data += dom(dom_object).text()

        sleep(1)
        return web_data
1343078
def get_from_archive(self, feed_id):
        
        file_path = self.feed_repository + '/' + str(feed_id) + '.xml'
        if not os.path.isfile(file_path):
            raise FeedArchiveNotFound()
        with open(file_path, 'r') as rec_file:
            return rec_file.read()
995915
def extract_data(self):
        
        content_type = 'Exception in extracting content-type from header'
        try:
            content_type = self.response.headers.get('content-type', '')
            if 'application/json' in content_type:
                self.raw_data = self.data = self.response.json()
            elif self.prepared_request.headers.get(
                    'Accept', '') == 'application/json':
                try:
                    self.raw_data = self.data = self.response.json()
                except Exception as e:
                    self.raw_data = self.data = self.response.content
            else:
                self.raw_data = self.data = self.response.content

            if isinstance(self.data,
                          dict) and 'server_timing' in self.data.keys():
                self._server_timing = self.data.pop(
                    'server_timing', )  # this is internal timing debug info
                self.raw_data = self.data = self.data['original_response']

        except Exception as e:
            self._timestamps.setdefault('exception', time.time())
            self.error = ApiError(
                "Invalid Response Data with content type: " + content_type,
                status_code=self.status_code,
                status_description=str(e),
                http_body=self.response.content,
                url=self.response.url)
            raise Exception(self.error)
252032
def http_request(url)
      url = URIParser.new(url).with_scheme

      CLI.put_header(url)
      CLI.log "Beginning HTTP request for #{url}"
      response = HTTParty.get(url, headers: { "User-Agent" => "news-scraper-#{NewsScraper::VERSION}" })

      raise ResponseError.new(
        error_code: response.code,
        message: response.message,
        url: url
      ) unless response.code == 200

      CLI.log "#{response.code} - #{response.message}. Request successful for #{url}"
      CLI.put_footer

      if block_given?
        yield response
      else
        response
      end
    end
124383
def extract_response(raw_response):
    
    data = urlread(raw_response)

    if is_success_response(raw_response):
        return data
    elif is_failure_response(raw_response):
        raise RemoteExecuteError(data)
    elif is_invalid_response(raw_response):
        raise InvalidResponseError(data)
    else:
        raise UnknownStatusError(data)
1023987
def get_feed_list(opml_obj: OPML) -> List[str]:
    """"""
    rv = list()

    def collect(obj):
        for outline in obj.outlines:
            if outline.type == 'rss' and outline.xml_url:
                rv.append(outline.xml_url)

            if outline.outlines:
                collect(outline)

    collect(opml_obj)
    return rv
707695
def import_opml(url):
    """"""
    # Test if URL given is local, then open, parse out feed urls,
    # add feeds, set text= to aliases and report success, list feeds added
    from bs4 import BeautifulSoup
    try:
        f = file(url).read()
    except IOError:
        f = requests.get(url).text
    soup = BeautifulSoup(f, "xml")
    links = soup.find_all('outline', type="rss" or "pie")
    # This is very slow, might cache this info on add
    for link in links:
        # print link
        add_feed(link['xmlUrl'])
        print("Added " + link['text'])
712251
def extract(self, item):
        
        article_candidate = ArticleCandidate()
        article_candidate.extractor = self._name()

        article = Article('')
        article.set_html(item['spider_response'].body)
        article.parse()
        article_candidate.title = article.title
        article_candidate.description = article.meta_description
        article_candidate.text = article.text
        article_candidate.topimage = article.top_image
        article_candidate.author = article.authors
        if article.publish_date is not None:
            try:
                article_candidate.publish_date = article.publish_date.strftime('%Y-%m-%d %H:%M:%S')
            except ValueError as exception:
                self.log.debug('%s: Newspaper failed to extract the date in the supported format,'
                              'Publishing date set to None' % item['url'])
        article_candidate.language = article.meta_lang

        return article_candidate
186464
def extract(self, url=None, raw_html=None):
        
        cc = CrawlCandidate(self.config, url, raw_html)
        return self.crawl(cc)
162907
public static void returnObjectResponse(HttpServletResponse response, Object obj) {
		if (response == null) {
			return;
		}
		PrintWriter out = null;
		try {
			response.setStatus(HttpServletResponse.SC_OK);
			response.setContentType(MediaType.APPLICATION_JSON);
			out = response.getWriter();
			ParaObjectUtils.getJsonWriter().writeValue(out, obj);
		} catch (Exception ex) {
			logger.error(null, ex);
		} finally {
			if (out != null) {
				out.close();
			}
		}
	}
1216215
def parse_response(response):
    
    if response.headers.get('Content-Type', JSON_TYPE).startswith(JSON_TYPE):
        return ResponseObject(json.loads(response.body))
    else:
        return response.body
845026
def parseRest(self, response):
        
        body = json.loads(response)

        try:
            error = body['error']['description']
            code = body['error']['code']
        except Exception:
            return body['data']
        else:
            raise ClickatellError(error, code);