Fetches HTML content from the given URL and then crawls it for Open Graph data.

@param   string  $url            URL to be crawled.

@return  Website
976110
public function loadUrl($url)
    {
        // Fetch HTTP content using Guzzle
        $response = $this->client->get($url);

        return $this->loadHtml($response->getBody()->__toString(), $url);
    }
471699
public function fetch( $url ) {
		try {
			$new_h5p_id = \H5P_Plugin::get_instance()->fetch_h5p( $url );
		} catch ( \Throwable $e ) {
			$new_h5p_id = 0;
		}
		return $new_h5p_id;
	}
52483
public function get($url)
    {
        $ch = curl_init($url);

        if ( ! ini_get('open_basedir')) {
            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        }

        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);

        $content = curl_exec($ch);
        if ($content === false) {
            // there was a problem
            $error = curl_error($ch);
            throw new CurlException('Error retrieving "'.$url.'" ('.$error.')');
        }

        return $content;
    }
736654
protected static function getUrl($url)
    {
        Craft::debug('Fetching JSON from: '.$url, __METHOD__);

        $ch = curl_init();
        curl_setopt_array(
            $ch,
            [
                CURLOPT_URL => $url,
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_MAXREDIRS => 5,
                CURLOPT_CONNECTTIMEOUT => 5,
            ]
        );
        $data = curl_exec($ch);

        $errorNumber = curl_errno($ch);
        $errorMessage = curl_error($ch);
        curl_close($ch);

        if ($errorNumber > 0) {
            Craft::warning('cURL got an error: '.$errorNumber.', '.$errorMessage, __METHOD__);
            return false;
        }

        return $data;
    }
520434
public static function tumblr(Url $url)
    {
        if (($value = $url->getQueryParameter('z'))) {
            return Url::create($value);
        }

        return $url;
    }
199724
static public function httpGet($url)
    {
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false);
        curl_setopt($curl, CURLOPT_SSLVERSION, 1);
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_TIMEOUT, 30);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
        list($content, $status) = array(curl_exec($curl), curl_getinfo($curl), curl_close($curl));
        return (intval($status["http_code"]) === 200) ? $content : false;
    }
808047
public function request($url)
    {
        $curlOpts = array_replace($this->curlOpts, [CURLOPT_URL => $url]);

        $ch = curl_init();
        foreach ($curlOpts as $opt => $optVal) {
            curl_setopt($ch, $opt, $optVal);
        }

        $resp = curl_exec($ch);
        $info = (object) curl_getinfo($ch);
        curl_close($ch);

        //Check response
        $httpCode = (string) $info->http_code;
        if ($httpCode{0} != '2') {
            $msg = sprintf('HTTP Request Failed (code %s): %s', $info->http_code, $resp);
            throw new HttpException($resp, $msg, $httpCode);
        } elseif (strlen(trim($resp)) == 0) {
            throw new HttpException($resp, 'HTTP Response Empty');
        }

        return $resp;
    }
1061603
public function parse($url)
    {
        $content = $this->performRequest($url)->getBody()->getContents();

        $content = $this->convertEncodingToUTF8($content);

        return $this->getHtmlDomParser($content);
    }
1040982
public function link($url)
    {
        $this->element = 'a';
        if (is_string($url)) {
            $this->setAttr('href', $url);

            return $this;
        }
        $this->setAttr('href', $this->url($url));

        return $this;
    }
1149346
public function crawl(string $url, string $rawHTML = null): ?Article {
        $article = new Article();

        $parseCandidate = Helper::getCleanedUrl($url);

        $xmlInternalErrors = libxml_use_internal_errors(true);

        if (empty($rawHTML)) {
            $guzzle = new \GuzzleHttp\Client();
            $response = $guzzle->get($parseCandidate->url, $this->config()->get('browser'));
            $article->setRawResponse($response);
            $rawHTML = $response->getBody()->getContents();
        }

        // Generate document
        $doc = $this->getDocument($rawHTML);

        // Set core mutators
        $article->setFinalUrl($parseCandidate->url);
        $article->setDomain($parseCandidate->parts->host);
        $article->setLinkhash($parseCandidate->linkhash);
        $article->setRawHtml($rawHTML);
        $article->setDoc($doc);
        $article->setRawDoc(clone $doc);

        // Pre-extraction document cleaning
        $this->modules('cleaners', $article);

        // Extract content
        $this->modules('extractors', $article);

        // Post-extraction content formatting
        $this->modules('formatters', $article);

        libxml_use_internal_errors($xmlInternalErrors);

        return $article;
    }
812696
public function extract($dom, $url)
    {
        $this->resetUrls();
        $crawler = new DomCrawler($dom, $url);
        $links = $crawler->filter('body a')->links();

        foreach ($links as $link) {
            if (!in_array($link->getUri(), $this->urls)) {
                $this->urls[] = $link->getUri();
            }
        }

        return $this->urls;
    }
287817
public function downloadAttachment($url)
	{
		$result = $this->client->sendRequest(
			self::REQUEST_GET,
			$url,
			array(),
			null,
			$this->authentication,
			true,
			false
		);

		return $result;
	}
657345
public function get(string $url): string
    {
        $ch = curl_init($url);

        if ( ! ini_get('open_basedir')) {
            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        }

        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_VERBOSE, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36');
        curl_setopt($ch, CURLOPT_URL, $url);

        $content = curl_exec($ch);
        if ($content === false) {
            // there was a problem
            $error = curl_error($ch);
            throw new CurlException('Error retrieving "'.$url.'" ('.$error.')');
        }

        return $content;
    }
1336893
public function download($url)
    {
        $context = $this->getContext($url);

        return file_get_contents($url, false, $context);
    }
829119
protected function getUrlContents($url)
    {
        $client = $this->getGuzzleClient();

        return $client->get($url)->getBody()->getContents();
    }
548823
public static function url($url){

		$url = static::_nullValue($url);
		if(!$url){

			return $url;
		}

		$query = parse_url($url, PHP_URL_QUERY);
		if(!$query){

			return $url;
		}

		parse_str($query, $query_parts);

		// First urldecode to be sure that we do not re-encode the query parts.
		$query_parts = array_map('static::_nested_urldecode', $query_parts);

		// http_build_query will take care of the urlencode.
		return strtok($url, '?') . '?' . http_build_query($query_parts);
	}
183867
private static function get(string $url)
    {
        $cu = curl_init();
        curl_setopt_array(
            $cu,
            [
                CURLOPT_URL            => $url,
                CURLOPT_RETURNTRANSFER => 1,
                CURLOPT_FOLLOWLOCATION => 1,
                CURLOPT_USERAGENT      => 'Torrent Parser PHP',
            ]
        );
        $response = curl_exec($cu);
        $response = str_replace('&nbsp;', ' ', $response);
        $xml = simplexml_load_string($response);

        return self::xml2array($xml)['channel'][0]['item'];
    }