Fetch a feed for the specified URL.

@param   string  $url The URL to fetch
@return  \moodle_simplepie
423766
protected function fetch_feed(string $url) : \moodle_simplepie {
        // Fetch the rss feed, using standard simplepie caching so feeds will be renewed only if cache has expired.
        \core_php_time_limit::raise(60);

        $feed = new \moodle_simplepie();

        // Set timeout for longer than normal to be agressive at fetching feeds if possible..
        $feed->set_timeout(40);
        $feed->set_cache_duration(0);
        $feed->set_feed_url($url);
        $feed->init();

        return $feed;
    }
311391
protected function request($url)
    {
        if (null === $this->httpClient) {
            $this->httpClient = new HttpClient();
        }

        return $this->httpClient
            ->get($url)
            ->getBody();
    }
1164729
public function pollFeed($url)
    {
        $self = $this;
        $eventEmitter = $this->getEventEmitter();
        $logger = $this->getLogger();
        $logger->info('Sending request for feed URL', array('url' => $url));
        $request = new HttpRequest(array(
            'url' => $url,
            'resolveCallback' => function($data) use ($url, $self) {
                $self->processFeed($url, $data);
            },
            'rejectCallback' => function($error) use ($url, $self) {
                $self->processFailure($url, $error);
            }
        ));
        $eventEmitter->emit('http.request', array($request));
    }
125009
public function get($url, $parameters = []) {
        $this->setOption(CURLOPT_HTTPGET, true);
        $this->setRequestMethod('GET');
        $this->setURL($this->buildURL($url, $parameters));

        return $this->run();
    }
214884
protected function getRequest($url) {
        $Request = new Request();
        $Request->setTransport(Request::TRANSPORT_CURL);
        return $Request->setUrl($url)
            ->setConnectTimeout($this->getConnectTimeout())
            ->setTimeout($this->getRequestTimeout())
            ->setContentTypeCode(Request::CONTENT_TYPE_JSON);
    }
679289
function get($url) {
      $snoopy = $this->getSnoopyInstance();
      $snoopy->fetch($url);
      return $this->processSnoopyResponse($snoopy);
    }
767298
public function fetch($url, array $params = array())
    {
        $params = array_merge(array(
            'curl' => array(),
            'fopen' => array(),
        ), $params);

        if (function_exists('curl_init') && $this->config['prefer_curl']) {
            return $this->curl($url, $params['curl']);
        }

        return $this->fileGetContents($url, $params['fopen']);
    }
890474
public static function get(string $url)
	{
		$ch = curl_init($url);

		curl_setopt_array(
			$ch,
			[
				CURLOPT_HEADER         => 1,
				CURLOPT_RETURNTRANSFER => 1,
				CURLOPT_SSL_VERIFYPEER => 0,
			]
		);

		$response = curl_exec($ch);

		$responseCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
		if ($responseCode != 200) {
			throw new HttpException('Could not reach ' . $url, $responseCode);
		}
		$contentType = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
		$headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);

		curl_close($ch);

		$response = substr($response, $headerSize);

		if (is_null($response)) {
			throw new HttpException('You are exceeding the API request limit.  Please wait longer between requests.', 400);
		}

		if(stripos($contentType, 'application/json') !== false) {
			$response = json_decode($response, true);
		}

		return $response;
	}
1353756
static function _fetchURL($url)
    {
        if (self::$disable_url_fetch) {
            return false;
        }

        $parts = parse_url($url);
        $data = '';
        switch ($parts['scheme']) {
            case 'http':
                $fsock = @fsockopen($parts['host'], isset($parts['port']) ? $parts['port'] : 80);
                if (!$fsock) {
                    return false;
                }
                fputs($fsock, "GET $parts[path] HTTP/1.0\r\n");
                fputs($fsock, "Host: $parts[host]\r\n\r\n");
                $line = fgets($fsock, 1024);
                if (strlen($line) < 3) {
                    return false;
                }
                preg_match('#HTTP/1.\d (\d{3})#', $line, $temp);
                if ($temp[1] != '200') {
                    return false;
                }

                // skip the rest of the headers in the http response
                while (!feof($fsock) && fgets($fsock, 1024) != "\r\n") {
                }

                while (!feof($fsock)) {
                    $data.= fread($fsock, 1024);
                }

                break;
            //case 'ftp':
            //case 'ldap':
            //default:
        }

        return $data;
    }
1336893
public function download($url)
    {
        $context = $this->getContext($url);

        return file_get_contents($url, false, $context);
    }
127335
public function loadUrl($url)
    {
        $str = $this->httpClient->get($url)->body;
        return $this->loadString($str, $url);
    }
434941
protected function get($url)
    {
        $request = $this->http->get(
            $url,
            array('User-Agent' => self::USER_AGENT.' v.'.self::VERSION)
        );

        $response = $request->send();

        if (!$response->isSuccessful()) {
            $this->log('fail with '.$response->getStatusCode().' http code in url "'.$url.'" ');

            return;
        }
        $this->logger->log(LogLevel::INFO, 'retrieved url "'.$url.'" ');

        return $response->getBody();
    }
117939
public function getRequestUrl($url)
    {
        if ($this->getPerPage() && $this->getPage()) {
            return $url . $this->getRequestPageInfo();
        }

        return $url;
    }
52483
public function get($url)
    {
        $ch = curl_init($url);

        if ( ! ini_get('open_basedir')) {
            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        }

        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);

        $content = curl_exec($ch);
        if ($content === false) {
            // there was a problem
            $error = curl_error($ch);
            throw new CurlException('Error retrieving "'.$url.'" ('.$error.')');
        }

        return $content;
    }
1140155
public function fetch($url)
    {
        if ($this->cache && $this->cache->contains($url)) {
            return $this->cache->fetch($url);
        } else {
            $result = $this->doFetch($url);

            if ($this->cache && $result) {
                $this->cache->save($url, $result);
            }

            return $result;
        }
    }
417561
private function doFetchContent($url)
    {
        $url = $this->validateUrl($url);
        $siteConfig = $this->configBuilder->buildFromUrl($url);

        $this->logger->info('Fetching url: {url}', ['url' => $url]);

        $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);

        $effectiveUrl = $response['effective_url'];
        $effectiveUrl = str_replace(' ', '%20', $effectiveUrl);
        if (!$this->isUrlAllowed($effectiveUrl)) {
            throw new \Exception(sprintf('Url "%s" is not allowed to be parsed.', $effectiveUrl));
        }

        // check if action defined for returned Content-Type, like image, pdf, audio or video
        $mimeInfo = $this->getMimeActionInfo($response['all_headers']);
        $infos = $this->handleMimeAction($mimeInfo, $effectiveUrl, $response);
        if (\is_array($infos)) {
            return $infos;
        }

        $html = $this->convert2Utf8($response['body'], $response['all_headers']);

        $this->logger->debug('Fetched HTML', ['html' => $html]);

        // Remove empty lines to avoid runaway evaluation of following regex
        // on badly coded websites
        $re = '/^[ \t]*[\r\n]+/m';
        $html = preg_replace($re, '', $html);

        // Remove empty nodes (except iframe)
        $re = '/<(?!iframe)([^>\s]+)[^>]*>(?:<br \/>|&nbsp;|&thinsp;|&ensp;|&emsp;|&#8201;|&#8194;|&#8195;|\s)*<\/\1>/m';
        $html = preg_replace($re, '', $html);

        $this->logger->debug('HTML after regex empty nodes stripping', ['html' => $html]);

        // some non utf8 enconding might be breaking after converting to utf8
        // when it happen the string (usually) starts with this character
        // in that case, we'll take the default response instead of the utf8 forced one
        if (0 === strpos(utf8_encode($response['body']), 'ÿþ')) {
            $html = $response['body'];
        }

        $ogData = $this->extractOpenGraph($html, $effectiveUrl);

        $this->logger->info('Opengraph data: {ogData}', ['ogData' => $ogData]);

        // @TODO: log raw html + headers

        // check site config for single page URL - fetch it if found
        $isSinglePage = false;
        if ($this->config['singlepage'] && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
            $isSinglePage = true;
            $effectiveUrl = $singlePageResponse['effective_url'];

            // check if action defined for returned Content-Type
            $mimeInfo = $this->getMimeActionInfo($singlePageResponse['all_headers']);
            $infos = $this->handleMimeAction($mimeInfo, $effectiveUrl, $singlePageResponse);
            if (\is_array($infos)) {
                return $infos;
            }

            $html = $this->convert2Utf8($singlePageResponse['body'], $singlePageResponse['all_headers']);
            $this->logger->info('Retrieved single-page view from "{url}"', ['url' => $effectiveUrl]);

            unset($singlePageResponse);
        }

        $this->logger->info('Attempting to extract content');

        $extractResult = $this->extractor->process($html, $effectiveUrl);
        $readability = $this->extractor->readability;
        $contentBlock = $this->extractor->getContent();
        $extractedTitle = $this->extractor->getTitle();
        $extractedLanguage = $this->extractor->getLanguage();
        $extractedDate = $this->extractor->getDate();
        $extractedAuthors = $this->extractor->getAuthors();

        // in case of no language were found, try using headers Content-Language
        if (empty($extractedLanguage) && !empty($response['all_headers']['content-language'])) {
            $extractedLanguage = $response['all_headers']['content-language'];
        }

        // Deal with multi-page articles
        $isMultiPage = (!$isSinglePage && $extractResult && null !== $this->extractor->getNextPageUrl());
        if ($this->config['multipage'] && $isMultiPage) {
            $this->logger->info('Attempting to process multi-page article');
            // store first page to avoid parsing it again (previous url content is in `$contentBlock`)
            $multiPageUrls = [$effectiveUrl];
            $multiPageContent = [];

            while ($nextPageUrl = $this->extractor->getNextPageUrl()) {
                $this->logger->info('Processing next page: {url}', ['url' => $nextPageUrl]);
                // If we've got URL, resolve against $url
                $nextPageUrl = $this->makeAbsoluteStr($effectiveUrl, $nextPageUrl);
                if (!$nextPageUrl) {
                    $this->logger->info('Failed to resolve against: {url}', ['url' => $effectiveUrl]);
                    $multiPageContent = [];
                    break;
                }

                // check it's not what we have already!
                if (\in_array($nextPageUrl, $multiPageUrls, true)) {
                    $this->logger->info('URL already processed');
                    $multiPageContent = [];
                    break;
                }

                // it's not, store it for later check & so let's attempt to fetch it
                $multiPageUrls[] = $nextPageUrl;

                $response = $this->httpClient->fetch($nextPageUrl, false, $siteConfig->http_header);

                // make sure mime type is not something with a different action associated
                $mimeInfo = $this->getMimeActionInfo($response['all_headers']);

                if (isset($mimeInfo['action'])) {
                    $this->logger->info('MIME type requires different action');
                    $multiPageContent = [];
                    break;
                }

                $extracSuccess = $this->extractor->process(
                    $this->convert2Utf8($response['body'], $response['all_headers']),
                    $nextPageUrl
                );

                if (!$extracSuccess) {
                    $this->logger->info('Failed to extract content');
                    $multiPageContent = [];
                    break;
                }

                $multiPageContent[] = clone $this->extractor->getContent();
            }

            // did we successfully deal with this multi-page article?
            if (empty($multiPageContent)) {
                $this->logger->info('Failed to extract all parts of multi-page article, so not going to include them');
                $page = $readability->dom->createElement('p');
                $page->innerHTML = '<em>This article appears to continue on subsequent pages which we could not extract</em>';
                $multiPageContent[] = $page;
            }

            foreach ($multiPageContent as $page) {
                $page = $contentBlock->ownerDocument->importNode($page, true);
                $contentBlock->appendChild($page);
            }

            unset($multiPageUrls, $multiPageContent, $nextPageUrl, $page);
        }

        $res = [
            'status' => $response['status'],
            'html' => $this->config['error_message'],
            'title' => $extractedTitle ?: $this->config['error_message_title'],
            'language' => $extractedLanguage,
            'date' => $extractedDate,
            'authors' => $extractedAuthors,
            'url' => $effectiveUrl,
            'content_type' => isset($mimeInfo['mime']) ? $mimeInfo['mime'] : '',
            'open_graph' => $ogData,
            'native_ad' => $this->extractor->isNativeAd(),
            'all_headers' => $response['all_headers'],
        ];

        // if we failed to extract content...
        if (!$extractResult || null === $contentBlock) {
            $this->logger->info('Extract failed');

            return $res;
        }

        $res['html'] = $this->cleanupHtml($contentBlock, $effectiveUrl);

        $this->logger->info('Returning data (most interesting ones): {data}', ['data' => ['html' => '(only length for debug): ' . \strlen($res['html'])] + $res]);

        return $res;
    }
1346356
public function callGetApi($url)
    {
        $this->registerApi($url, __FUNCTION__, func_get_args());
        return $this->httpGetForJson($url);
    }