Loads HTML content using CURL.

@param string $url URL of the page.

@return string HTML content (on success).
@throws Exception If content loading error occurred.
559530
private static function loadContent($url) {
        //TODO: implement URL content load functionality with CURL and proxy

        $ch = curl_init($url);
        $options = array(
            CURLOPT_RETURNTRANSFER => true
        );
        curl_setopt_array($ch, $options);

        $content = curl_exec($ch);

        curl_close($ch);

        if ($content === false) {
            throw new Exception("Can't get content from " . $url);
        }
        return $content;
    }
151701
public static function parse($url)
    {
        $scheme = substr($url, 0, strpos($url, ':'));
        if (strpos($scheme, '+')) {
            $scheme = substr($scheme, 0, strpos($scheme, '+'));
        }

        if (!$scheme) {
            throw new \Exception(sprintf('The url \'%s\' could not be parsed', $url));
        }

        if (isset(static::$schemeMap[$scheme])) {
            $className = static::$schemeMap[$scheme];
        } else {
            $className  = static::currentNamespace() . '\\' . ucfirst($scheme) . 'Dsn';
            if (!class_exists($className)) {
                $className  = static::currentClass();
            }
        }
        return new $className($url);
    }
853724
private function fetchImageContentFromURL($url)
    {
        $data = null;

        if ($stream = fopen($url, 'r')) {
            $data = stream_get_contents($stream);
            fclose($stream);
        }

        return $data;
    }
658743
function ParseUrl($url) {
	    $content = null;

    	$this->Clear();

		$this->url = htmlspecialchars_decode($url);

	    ob_start();
	    if (function_exists('curl_init')) {
		    //curl is the preferred function
		    $crl = curl_init();
		    curl_setopt($crl, CURLOPT_URL, $url);
		    curl_setopt($crl, CURLOPT_RETURNTRANSFER, 1);
		    curl_setopt($crl, CURLOPT_CONNECTTIMEOUT, 5);
		    curl_setopt($crl, CURLOPT_TIMEOUT, 20);
		    $FileData = curl_exec($crl);
		    $FileInfo = curl_getinfo($crl);
		    curl_close($crl);
		    if ($FileInfo['http_code'] != 200) {
			    throw new \HttpRequestException($FileInfo['http_code'] . ' error for file: ' . $url);
		    }
	    } else {
		    //see if we can use fopen to get file
		    $fopen_check = ini_get('allow_url_fopen');
		    if (!empty($fopen_check)) {
			    $content = file_get_contents($url);
		    } else {
			    throw new \RuntimeException('CURL_DISABLED');
		    }
	    }
	    ob_end_clean();

		if(strlen($content)) {
			$this->modifyContent($content);
			$this->ParseStr($content);
			return true;
		} else {
			return false;
		}
	}
873548
public function parseURL($url)
	{
		try {
			$u = parse_url($url);
			if(isset($u["query"]))
	        	$this->parseQueryString($u["query"]);
	        if(isset($u["fragment"]))
	        	$this->parseQueryString($u["fragment"]);
	    } catch (Exception $e) {
	    } 
	}
1149048
public function getContent($url)
    {
        $cache = $this->getCache($url);
        if ($cache->isCached('content')) {
            $content = $cache->retrieve('content');
        } else {
            $content = $this->downloader->getContent($url);
            foreach ($this->getStrippedHtmlTags() as $strippedHtmlTag) {
                $content = preg_replace("/<{$strippedHtmlTag}.*?>.*?<\/{$strippedHtmlTag}>/is", "", $content);
            }
            $cache->store('content', $content);
        }
        return $content;
    }
1326658
public function download($url)
    {
        if (!$this->curl) {
            $this->curl = curl_init();
            curl_setopt($this->curl, CURLOPT_HTTPHEADER, array(
                'User-Agent: gender-api-client/1.0',
                'Content-Type: application/json'
            ));
            curl_setopt($this->curl, CURLOPT_TIMEOUT, 5);
            curl_setopt($this->curl, CURLOPT_CONNECTTIMEOUT, 5);
            curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, true);

            if ($this->proxyHost) {
                curl_setopt($this->curl, CURLOPT_PROXY, $this->proxyHost);
                curl_setopt($this->curl, CURLOPT_PROXYPORT, $this->proxyPort);
            }
        }

        curl_setopt($this->curl, CURLOPT_URL, $url);

        $response = @curl_exec($this->curl);
        $responseCode = curl_getinfo($this->curl, CURLINFO_RESPONSE_CODE);

        if ($response === false || $responseCode != 200) {
            $lastError = curl_error($this->curl);
            throw new NetworkErrorException($lastError);
        }

        return $response;
    }
284015
public function get($url) {
		$reporting = error_reporting(0);
		$context = $this->_createContext();
		$contents = file_get_contents($url, false, $context);
		error_reporting($reporting);

		if ($contents === false) {
			$code = isset($http_response_header[0])
				? $this->_extractHttpCode($http_response_header[0])
				: $this->_defaultCode;

			throw new Exception($url, $code);
		}

		return $contents;
	}
828599
private function loadXML($url) {
		if (!file_exists($url)) {
			throw new \n2n\log4php\LoggerException("File [$url] does not exist.");
		}

		libxml_clear_errors();
		$oldValue = libxml_use_internal_errors(true);
		
		// Load XML
		$xml = @simplexml_load_file($url);
		if ($xml === false) {
			
			$errorStr = "";
			foreach(libxml_get_errors() as $error) {
				$errorStr .= "[" . $error->file . "] line " . $error->line . " : " . $error->message;
			}
			
			throw new \n2n\log4php\LoggerException("Error loading configuration file: " . trim($errorStr));
		}
		
		libxml_clear_errors();
		libxml_use_internal_errors($oldValue);
		
		return $xml;
	}
1007296
public function url(string $url = null) : self
    {
        if (! $url) {
            $url = null;

            $httpHost = getenv('APP_URL'); // Has to start with a protocol - for example "http://"!

            if ($httpHost === false) {
                $url = 'http';

                // Quick and dirty
                if (isset($_SERVER['HTTPS'])) {
                    $url .= 's';
                }

                $url .= '://';

                $httpHost = isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : 'localhost/';
            }  

            $requestUri = isset($_SERVER['REQUEST_URI']) ? $_SERVER['REQUEST_URI'] : '';
            
            $safeRequestURI = htmlentities(strip_tags(urldecode($requestUri)));

            $url .= "{$httpHost}{$safeRequestURI}";
        } 

        if ($this->validate and ! filter_var($url, FILTER_VALIDATE_URL)) {
            throw new Exception("Open Graph: Invalid URL '{$url}'");
        }

        $this->forget('url');

        $this->tags[] = $this->createTag('url', $url);

        return $this;
    }
1184217
public function getDomDocument($url)
    {
        $dom = new \DOMDocument();
        @$dom->loadHTML($this->getContent($url));
        return $dom;
    }
95742
protected function loadRemoteFile($url)
    {
        $this->file = tempnam($this->tmp, 'jit_rmt_');

        if (!function_exists('curl_init')) {

            if (!$contents = file_get_contents($url)) {
                return false;
            }

            file_put_contents($contents, $this->file);

            return $this->file;

        }

        $handle = fopen($this->file, 'w');

        if (!$this->fetchFile($handle, $url)) {
            fclose($handle);
            return false;
        }

        fclose($handle);

        return $this->file;
    }
1351092
public function fetchInfo($url)
    {
        $response = null;

        if (function_exists('curl_init')) {
            $ch = curl_init();
            curl_setopt($ch, CURLOPT_URL, $url);
            curl_setopt($ch, CURLOPT_TIMEOUT, 30);
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
            curl_setopt($ch, CURLOPT_USERAGENT, 'LarageoPlugin Package v1.0');
            $response = curl_exec($ch);
            curl_close($ch);
        } elseif (ini_get('allow_url_fopen')) {
            $response = file_get_contents($url, 'r');
        } else {
            throw new \Exception('LarageoPlugin requires the CURL PHP extension or allow_url_fopen set to 1!');
        }

        $response = json_decode($response);

        if (empty($response)) {
            throw new \Exception('Ops! The data is empty! Is '.$url.' acessible?');
        }

        if (isset($response->geoplugin_status) && $response->geoplugin_status == 404) {
            throw new \Exception('Ops! Your request returned a 404 error! Is '.$url.' acessible?');
        }

        return $response;
    }
493099
protected function getDocument( $content, $url )
	{
		$doc = new DOMDocument();
		ob_start();
		if( !@$doc->loadHTML( $content ) )
		{
			$content	= ob_get_clean();
			if( $content )
				$this->errors[$url]	= $content;
			throw new RuntimeException( 'Error reading HTML.' );
		}
		ob_end_clean();
		return $doc;
	}
1336893
public function download($url)
    {
        $context = $this->getContext($url);

        return file_get_contents($url, false, $context);
    }
657345
public function get(string $url): string
    {
        $ch = curl_init($url);

        if ( ! ini_get('open_basedir')) {
            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        }

        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_VERBOSE, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36');
        curl_setopt($ch, CURLOPT_URL, $url);

        $content = curl_exec($ch);
        if ($content === false) {
            // there was a problem
            $error = curl_error($ch);
            throw new CurlException('Error retrieving "'.$url.'" ('.$error.')');
        }

        return $content;
    }
417561
private function doFetchContent($url)
    {
        $url = $this->validateUrl($url);
        $siteConfig = $this->configBuilder->buildFromUrl($url);

        $this->logger->info('Fetching url: {url}', ['url' => $url]);

        $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);

        $effectiveUrl = $response['effective_url'];
        $effectiveUrl = str_replace(' ', '%20', $effectiveUrl);
        if (!$this->isUrlAllowed($effectiveUrl)) {
            throw new \Exception(sprintf('Url "%s" is not allowed to be parsed.', $effectiveUrl));
        }

        // check if action defined for returned Content-Type, like image, pdf, audio or video
        $mimeInfo = $this->getMimeActionInfo($response['all_headers']);
        $infos = $this->handleMimeAction($mimeInfo, $effectiveUrl, $response);
        if (\is_array($infos)) {
            return $infos;
        }

        $html = $this->convert2Utf8($response['body'], $response['all_headers']);

        $this->logger->debug('Fetched HTML', ['html' => $html]);

        // Remove empty lines to avoid runaway evaluation of following regex
        // on badly coded websites
        $re = '/^[ \t]*[\r\n]+/m';
        $html = preg_replace($re, '', $html);

        // Remove empty nodes (except iframe)
        $re = '/<(?!iframe)([^>\s]+)[^>]*>(?:<br \/>|&nbsp;|&thinsp;|&ensp;|&emsp;|&#8201;|&#8194;|&#8195;|\s)*<\/\1>/m';
        $html = preg_replace($re, '', $html);

        $this->logger->debug('HTML after regex empty nodes stripping', ['html' => $html]);

        // some non utf8 enconding might be breaking after converting to utf8
        // when it happen the string (usually) starts with this character
        // in that case, we'll take the default response instead of the utf8 forced one
        if (0 === strpos(utf8_encode($response['body']), 'ÿþ')) {
            $html = $response['body'];
        }

        $ogData = $this->extractOpenGraph($html, $effectiveUrl);

        $this->logger->info('Opengraph data: {ogData}', ['ogData' => $ogData]);

        // @TODO: log raw html + headers

        // check site config for single page URL - fetch it if found
        $isSinglePage = false;
        if ($this->config['singlepage'] && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
            $isSinglePage = true;
            $effectiveUrl = $singlePageResponse['effective_url'];

            // check if action defined for returned Content-Type
            $mimeInfo = $this->getMimeActionInfo($singlePageResponse['all_headers']);
            $infos = $this->handleMimeAction($mimeInfo, $effectiveUrl, $singlePageResponse);
            if (\is_array($infos)) {
                return $infos;
            }

            $html = $this->convert2Utf8($singlePageResponse['body'], $singlePageResponse['all_headers']);
            $this->logger->info('Retrieved single-page view from "{url}"', ['url' => $effectiveUrl]);

            unset($singlePageResponse);
        }

        $this->logger->info('Attempting to extract content');

        $extractResult = $this->extractor->process($html, $effectiveUrl);
        $readability = $this->extractor->readability;
        $contentBlock = $this->extractor->getContent();
        $extractedTitle = $this->extractor->getTitle();
        $extractedLanguage = $this->extractor->getLanguage();
        $extractedDate = $this->extractor->getDate();
        $extractedAuthors = $this->extractor->getAuthors();

        // in case of no language were found, try using headers Content-Language
        if (empty($extractedLanguage) && !empty($response['all_headers']['content-language'])) {
            $extractedLanguage = $response['all_headers']['content-language'];
        }

        // Deal with multi-page articles
        $isMultiPage = (!$isSinglePage && $extractResult && null !== $this->extractor->getNextPageUrl());
        if ($this->config['multipage'] && $isMultiPage) {
            $this->logger->info('Attempting to process multi-page article');
            // store first page to avoid parsing it again (previous url content is in `$contentBlock`)
            $multiPageUrls = [$effectiveUrl];
            $multiPageContent = [];

            while ($nextPageUrl = $this->extractor->getNextPageUrl()) {
                $this->logger->info('Processing next page: {url}', ['url' => $nextPageUrl]);
                // If we've got URL, resolve against $url
                $nextPageUrl = $this->makeAbsoluteStr($effectiveUrl, $nextPageUrl);
                if (!$nextPageUrl) {
                    $this->logger->info('Failed to resolve against: {url}', ['url' => $effectiveUrl]);
                    $multiPageContent = [];
                    break;
                }

                // check it's not what we have already!
                if (\in_array($nextPageUrl, $multiPageUrls, true)) {
                    $this->logger->info('URL already processed');
                    $multiPageContent = [];
                    break;
                }

                // it's not, store it for later check & so let's attempt to fetch it
                $multiPageUrls[] = $nextPageUrl;

                $response = $this->httpClient->fetch($nextPageUrl, false, $siteConfig->http_header);

                // make sure mime type is not something with a different action associated
                $mimeInfo = $this->getMimeActionInfo($response['all_headers']);

                if (isset($mimeInfo['action'])) {
                    $this->logger->info('MIME type requires different action');
                    $multiPageContent = [];
                    break;
                }

                $extracSuccess = $this->extractor->process(
                    $this->convert2Utf8($response['body'], $response['all_headers']),
                    $nextPageUrl
                );

                if (!$extracSuccess) {
                    $this->logger->info('Failed to extract content');
                    $multiPageContent = [];
                    break;
                }

                $multiPageContent[] = clone $this->extractor->getContent();
            }

            // did we successfully deal with this multi-page article?
            if (empty($multiPageContent)) {
                $this->logger->info('Failed to extract all parts of multi-page article, so not going to include them');
                $page = $readability->dom->createElement('p');
                $page->innerHTML = '<em>This article appears to continue on subsequent pages which we could not extract</em>';
                $multiPageContent[] = $page;
            }

            foreach ($multiPageContent as $page) {
                $page = $contentBlock->ownerDocument->importNode($page, true);
                $contentBlock->appendChild($page);
            }

            unset($multiPageUrls, $multiPageContent, $nextPageUrl, $page);
        }

        $res = [
            'status' => $response['status'],
            'html' => $this->config['error_message'],
            'title' => $extractedTitle ?: $this->config['error_message_title'],
            'language' => $extractedLanguage,
            'date' => $extractedDate,
            'authors' => $extractedAuthors,
            'url' => $effectiveUrl,
            'content_type' => isset($mimeInfo['mime']) ? $mimeInfo['mime'] : '',
            'open_graph' => $ogData,
            'native_ad' => $this->extractor->isNativeAd(),
            'all_headers' => $response['all_headers'],
        ];

        // if we failed to extract content...
        if (!$extractResult || null === $contentBlock) {
            $this->logger->info('Extract failed');

            return $res;
        }

        $res['html'] = $this->cleanupHtml($contentBlock, $effectiveUrl);

        $this->logger->info('Returning data (most interesting ones): {data}', ['data' => ['html' => '(only length for debug): ' . \strlen($res['html'])] + $res]);

        return $res;
    }