Add a list of URLs to the queue, returns how many unique URLs were
actually added to the queue

@param array $urls
@return int
547252
public function addUrlsToCrawlerQueue(array $urls) {
        // TODO: remove this debug message
        if ($this->getCrawlerDebugEnabled()) {
            foreach ($urls as $url) {
                Mage::helper('turpentine/debug')->log(
                    'Adding URL to queue: %s', $url );
            }
        }
        $oldQueue = $this->_readUrlQueue();
        $newQueue = array_unique(array_merge($oldQueue, $urls));
        $this->_writeUrlQueue($newQueue);
        $diff = count($newQueue) - count($oldQueue);
        return $diff;
    }
896002
public function putBaiDu(array $urls) {
        return $this->getBaiDu()->setHeader([
            'Content-Type' => 'text/plain'
        ])->parameters($this->merge([
            'urls' => $urls
        ]))->encode(function ($data) {
            return implode("\n", $data['urls']);
        })->json();
    }
570193
public function numLinks()
    {
        if (!is_array($this->links()) && !($this->links() instanceof \Traversable)) {
            return 0;
        }

        $links = array_filter($this->links, function ($link) {
            if (isset($link['active']) && !$link['active']) {
                return false;
            }

            if (isset($link['required_acl_permissions'])) {
                $link['permissions'] = $link['required_acl_permissions'];
                unset($link['required_acl_permissions']);
            }

            if (isset($link['permissions'])) {
                if ($this->hasPermissions($link['permissions']) === false) {
                    return false;
                }
            }

            return true;
        });

        return count($links);
    }
639017
protected function getUrlStatus($url)
    {
        if($this->isValidUrl($url)) {
            ini_set('allow_url_fopen', 1);
            $url = @get_headers($url, 1);
            if($url) {
                $status = explode(' ',$url[0]);
                return (int) $status[1];
            }
        }
        else {
            return 404;
        }
    }
749160
public function countByParams($params = [])
    {
        $this->injectParams($params);
        $query = $this->newQuery('select');
        $query->addParams($params);

        return $this->countByQuery($query);
    }
197295
public function validate(array $urls)
    {
        if (!$this->options['enabled']) {
            return;
        }

        $master = curl_multi_init();
        $handlers = [];

        // Batch size can't be larger then number of urls
        $batchSize = min(count($urls), $this->options['batch_size']);
        for ($i = 0; $i < $batchSize; ++$i) {
            curl_multi_add_handle($master, $this->createCurlHandlerForUrl($urls[$i], $handlers));
        }

        do {
            while (($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM);

            if ($execrun != CURLM_OK) {
                break;
            }

            while ($done = curl_multi_info_read($master)) {
                $handler = $done['handle'];

                $this->doValidate($handlers[(int)$handler], $handler);

                if ($i < count($urls)) {
                    curl_multi_add_handle($master, $this->createCurlHandlerForUrl($urls[$i], $handlers));
                    ++$i;
                }

                curl_multi_remove_handle($master, $handler);
                curl_close($handler);
            }
        } while ($running);

        curl_multi_close($master);
    }
417414
public function getNextUrls($max = null){

        if($max){
            $max = $this->currentPage + $max;
        } else {
            $max = $this->totalPages;
        }//el

        return $this->getRangeUrls($this->currentPage + 1,$max);

    }
747199
public function getNumberOfShares($url)
    {
        $urlEncoded = urlencode($url);
        $data = array(
            'facebook' => 0,
            'facebookShare' => 0,
            'facebookLike' => 0,
            'facebookComment' => 0,
            'facebookClick' => 0,
            'twitter' => 0,
            'google' => 0,
            'pinterest' => 0,
        );

        // Facebook shares
        try {
            $tmp = file_get_contents('https://api.facebook.com/method/fql.query?format=json&query=select%20%20like_count,share_count,comment_count,click_count,total_count%20from%20link_stat%20where%20url=%22'.$urlEncoded.'%22');
            if ($tmp) {
                $tmpJson = json_decode($tmp, true);
                if (is_array($tmpJson) && count($tmpJson)) {
                    if (isset($tmpJson[0]['like_count'])) {
                        $retCache['facebookLike'] = $tmpJson[0]['like_count'];
                    }
                    if (isset($tmpJson[0]['comment_count'])) {
                        $retCache['facebookComment'] = $tmpJson[0]['comment_count'];
                    }
                    if (isset($tmpJson[0]['click_count'])) {
                        $retCache['facebookClick'] = $tmpJson[0]['click_count'];
                    }
                    if (isset($tmpJson[0]['share_count'])) {
                        $retCache['facebookShare'] = $tmpJson[0]['share_count'];
                    }
                    if (isset($tmpJson[0]['total_count'])) {
                        $retCache['facebook'] = $tmpJson[0]['total_count'];
                    }
                }
            }
        } catch (\Exception $e) {
        }

        // Twitter shares
        try {
            $tmp = file_get_contents('http://urls.api.twitter.com/1/urls/count.json?url='.$urlEncoded);
            if ($tmp) {
                $tmpJson = json_decode($tmp, true);
                if (isset($tmpJson['count'])) {
                    $data['twitter'] = $tmpJson['count'];
                }
            }
        } catch (\Exception $e) {
        }

        // Google+ shares
        try {
            $tmp = file_get_contents('https://plusone.google.com/_/+1/fastbutton?url='.$urlEncoded);
            if ($tmp) {
                @preg_match_all('#{c: (.*?),#si', $tmp, $matches);
                $ret = isset($matches[1][0]) && strlen($matches[1][0]) > 0 ? trim($matches[1][0]) : 0;
                if (0 != $ret) {
                    $data['google'] = str_replace('.0', '', $ret);
                }
            }
        } catch (\Exception $e) {
        }

        // Pinterest shares
        try {
            $tmp = file_get_contents('http://api.pinterest.com/v1/urls/count.json?callback=myCallback&url='.$urlEncoded);
            if ($tmp) {
                $tmp = trim(str_replace('myCallback({', '{', $tmp), ')');
                $tmpJson = json_decode($tmp, true);
                if (isset($tmpJson['count'])) {
                    $data['pinterest'] = $tmpJson['count'];
                }
            }
        } catch (\Exception $e) {
        }

        return $data;
    }
767492
public function getAlternateLinks($urls)
    {
        // determine default and current values
        $webspaceKey = $this->requestAnalyzer->getWebspace()->getKey();
        $currentPortal = $this->requestAnalyzer->getPortal();

        $defaultLocale = null;
        if (null !== $currentPortal && null !== ($defaultLocale = $currentPortal->getXDefaultLocalization())) {
            $defaultLocale = $defaultLocale->getLocale();
        }

        $result = [];
        foreach ($urls as $locale => $url) {
            // url = '/' means that there is no translation for this page
            // the only exception is the homepage where the requested resource-locator is '/'
            if ('/' !== $url || '/' === $this->requestAnalyzer->getResourceLocator()) {
                if ($locale === $defaultLocale) {
                    $result[] = $this->getAlternate($url, $webspaceKey, $locale, true);
                }
                $result[] = $this->getAlternate($url, $webspaceKey, $locale);
            }
        }

        return implode(PHP_EOL, $result);
    }
304140
protected function crawledBadUrls(): bool
    {
        return collect($this->urlsGroupedByStatusCode)->keys()->filter(function ($statusCode) {
            return ! $this->isSuccessOrRedirect($statusCode);
        })->count() > 0;
    }
580594
public function prefetchUrls(array $urls)
    {
        $req = array(
            'urls' => $urls,
        );

        $url = $this->server . '/v2/tune/prefetch';
        $body = json_encode($req);
        return $this->post($url, $body);
    }
671105
protected function addRequestVariablesToUrlParameters(array $urlParameters)
    {
        $urlParameters = parent::addRequestVariablesToUrlParameters($urlParameters);
        $urlParameters['name'] = $this->simplePageName;
        
        return $urlParameters;
    }
274717
public function addSegments($segments)
    {
        if ( ! $segments instanceof Segments) {
            $segments = new Segments($segments);
        }

        $currentSegments = $this->segments->getParts();
        $addSegments = $segments->getParts();

        $uri = clone $this;
        $uri->segments = new Segments(array_merge($currentSegments, $addSegments));

        return $uri;
    }
185207
public function addSites(array $list) : int
    {
        $count = $this->getObjectsCount();
        $nb_customer = $this->getCustomer();

        foreach ($list as $item) {
            if (!($item instanceof CNabuSite) &&
                is_numeric($nb_site_id = nb_getMixedValue($item, NABU_SITE_FIELD_ID))
            ) {
                $nb_site = $nb_customer->getSite($nb_site_id);
                if (!$nb_site instanceof CNabuSite) {
                    throw new ENabuCoreException(ENabuCoreException::ERROR_SITE_NOT_FOUND);
                }
            } elseif ($item instanceof CNabuSite) {
                $nb_site = $item;
                if (!$nb_site->validateCustomer($nb_customer)) {
                    throw new ENabuSecurityException(ENabuSecurityException::ERROR_CUSTOMER_NOT_OWNER);
                }
            } else {
                throw new ENabuCoreException(ENabuCoreException::ERROR_SITE_NOT_FOUND);
            }

            $nb_site->refresh(true, true);
            $this->nb_site_list->addItem($nb_site);
            $this->nb_language_list->merge($nb_site->getLanguages(true));
            $this->nb_role_list->merge($nb_site->getRoles(true));
        }

        return $this->getObjectsCount() - $count;
    }
1317098
private function getTrackerHosts($urls)
    {
        $hosts = array();
        foreach ($urls as $url) {
            $url = parse_url($url);
            if (isset($url['host'])) {
                $hosts[] = $url['host'];
            }
        }
        return $hosts;
    }
1332484
public function ping($urls, $publishDate = 0)
    {
        $this->checkSettings();

        $urls = (array)$urls;

        try {
            $response = $this->doRequest($urls, $publishDate);
        } catch (ClientException $e) {
            $xml = $this->getDecodedBody($e->getResponse()->getBody());

            if (isset($xml->error) && isset($xml->error->message)) {
                $errorMessage = (string) $xml->error->message;
                if (isset($xml->error->param) && isset($xml->error->value)) {
                    $errorMessage .= " (".$xml->error->param." is ".$xml->error->value.")";
                }
                throw new InvalidArgumentException($errorMessage);
            }
            return 0;
        }

        if (!$xml = $this->getDecodedBody($response->getBody())) {
            throw new SiteSearchPingerException("Wrong server response format");
        } elseif ($xml->getName() == 'empty-param') {
            // workaround for invalid request, with empty `urls`
            throw new InvalidUrlException("URL param is required");
        }

        // retrieve count of valid urls
        $addedCount = 0;
        if (isset($xml->added) && isset($xml->added['count'])) {
            $addedCount = (int)$xml->added['count'];
        }

        // check invalid urls and fill errors stack
        $this->invalidUrls = [];
        if (isset($xml->invalid)) {
            foreach ($xml->invalid as $invalid) {
                foreach ($invalid as $url) {
                    $this->invalidUrls[(string)$url] = $this->invalid[(string) $invalid['reason']];
                }
            }
        }

        return $addedCount;
    }
131981
public static function getEquityLinkCount($url = false)
    {
        $data = static::getCols('2048', $url);
        return (parent::noDataDefaultValue() == $data) ? $data :
            $data['uid'];
    }