Get the crawler URL queue from the cache

@return array
968397
protected function _readUrlQueue() {
        $readQueue = @unserialize(
            Mage::app()->loadCache(self::CRAWLER_URLS_CACHE_ID) );
        if ( ! is_array($readQueue)) {
            // This is the first time the queue has been read since the last
            // cache flush (or the queue is corrupt)
            // Returning an empty array here would be the proper behavior,
            // but causes the queue to not be saved on the full cache flush event
            return $this->getAllUrls();
        } else {
            return $readQueue;
        }
    }
367873
public function getUrls()
    {
        $urls = [];
        if ($this->entity->getUrls()) {
            foreach ($this->entity->getUrls() as $url) {
                $urls[] = $url;
            }
        }

        return $urls;
    }
647545
protected function getLinks()
	{
		$html = '';

		if (is_array($this->window['first'])) {
			$html .= $this->getUrlLinks($this->window['first']);
		}

		if (is_array($this->window['slider'])) {
			$html .= $this->getDots();
			$html .= $this->getUrlLinks($this->window['slider']);
		}

		if (is_array($this->window['last'])) {
			$html .= $this->getDots();
			$html .= $this->getUrlLinks($this->window['last']);
		}

		return $html;
	}
322237
protected function getCachedRoutes()
    {
        $ttl = (int)eZINI::instance( 'rest.ini' )->variable( 'CacheSettings', 'RouteApcCacheTTL' );

        if( self::$isRouteCacheCreated === false )
        {
            $options = array( 'ttl' => $ttl );
            ezcCacheManager::createCache( self::ROUTE_CACHE_ID, self::ROUTE_CACHE_PATH, 'ezpRestCacheStorageApcCluster', $options );
            self::$isRouteCacheCreated = true;
        }

        $cache = ezcCacheManager::getCache( self::ROUTE_CACHE_ID );
        $cacheKey = self::ROUTE_CACHE_KEY . '_' . ezpRestPrefixFilterInterface::getApiProviderName();
        if( ( $prefixedRoutes = $cache->restore( $cacheKey ) ) === false )
        {
            try
            {
                $prefixedRoutes = $this->doCreateRoutes();
                $cache->store( $cacheKey, $prefixedRoutes );
            }
            catch( Exception $e )
            {
                // Sometimes APC can miss a write. No big deal, just log it.
                // Cache will be regenerated next time
                ezpRestDebug::getInstance()->log( $e->getMessage(), ezcLog::ERROR );
            }
        }

        return $prefixedRoutes;
    }
547252
public function addUrlsToCrawlerQueue(array $urls) {
        // TODO: remove this debug message
        if ($this->getCrawlerDebugEnabled()) {
            foreach ($urls as $url) {
                Mage::helper('turpentine/debug')->log(
                    'Adding URL to queue: %s', $url );
            }
        }
        $oldQueue = $this->_readUrlQueue();
        $newQueue = array_unique(array_merge($oldQueue, $urls));
        $this->_writeUrlQueue($newQueue);
        $diff = count($newQueue) - count($oldQueue);
        return $diff;
    }
423583
protected function obtainCtrlRouterLinkTasks(): array
    {
        return [
            'searchRoute'     => \BFW\RunTasks::generateStepItem(
                $this->ctrlRouterInfos
            ),
            'checkRouteFound' => \BFW\RunTasks::generateStepItem(
                null,
                function() {
                    if ($this->ctrlRouterInfos->isFound === false) {
                        http_response_code(404);
                    }
                }
            ),
            'execRoute'       => \BFW\RunTasks::generateStepItem(
                $this->ctrlRouterInfos
            )
        ];
    }
87752
protected function getQueues() {
        if (!$this->_queues) {
            $this->_queues = new QueueList($this->version, $this->solution['sid']);
        }

        return $this->_queues;
    }
492246
public function getCrawlerStatus()
  {
    // Get crawler-status from file
    if ($this->write_status_to_file == true)
    {
      $this->crawlerStatus = PHPCrawlerUtils::deserializeFromFile($this->working_directory."crawlerstatus.tmp");
      if ($this->crawlerStatus == null) $this->crawlerStatus = new PHPCrawlerStatus();
    }
    
    return $this->crawlerStatus;
  }
101227
public function getQueues()
    {
        // Format the output to suit
        $queues = [];
        foreach ($this->queue->getQueues() as $queue) {
            $queues[] = $queue->name;
        }
        return $queues;
    }
1241638
public function getProcessReport()
  { 
    // Get current crawler-Status
    $CrawlerStatus = $this->crawlerStatus;
    
    // Create report
    $Report = new PHPCrawlerProcessReport();
    
    $Report->links_followed = $CrawlerStatus->links_followed;
    $Report->files_received = $CrawlerStatus->documents_received;
    $Report->bytes_received = $CrawlerStatus->bytes_received;
    $Report->process_runtime = PHPCrawlerBenchmark::getElapsedTime("crawling_process");
    
    if ($Report->process_runtime > 0)
      $Report->data_throughput = $Report->bytes_received / $Report->process_runtime;
    
    // Process abort-reason
    $Report->abort_reason = $CrawlerStatus->abort_reason;
    
    if ($CrawlerStatus->abort_reason == PHPCrawlerAbortReasons::ABORTREASON_TRAFFICLIMIT_REACHED)
      $Report->traffic_limit_reached = true;
    
    if ($CrawlerStatus->abort_reason == PHPCrawlerAbortReasons::ABORTREASON_FILELIMIT_REACHED)
      $Report->file_limit_reached = true;
    
    if ($CrawlerStatus->abort_reason == PHPCrawlerAbortReasons::ABORTREASON_USERABORT)
      $Report->user_abort = true;
    
    // Peak memory-usage
    if (function_exists("memory_get_peak_usage"))
      $Report->memory_peak_usage = memory_get_peak_usage(true);
    
    // Benchmark: Average server connect time
    if ($CrawlerStatus->sum_server_connects > 0)
      $Report->avg_server_connect_time = $CrawlerStatus->sum_server_connect_time / $CrawlerStatus->sum_server_connects;
      
    // Benchmark: Average server response time
    if ($CrawlerStatus->sum_server_responses > 0)
      $Report->avg_server_response_time = $CrawlerStatus->sum_server_response_time / $CrawlerStatus->sum_server_responses;
    
    // Average data tranfer time
    if ($CrawlerStatus->sum_data_transfer_time > 0)
      $Report->avg_proc_data_transfer_rate = $CrawlerStatus->unbuffered_bytes_read / $CrawlerStatus->sum_data_transfer_time;

    return $Report;
  }
236661
public function getQueues()
    {
        $queueNames = [];
        $queues = $this->queue->listQueues(['QueueNamePrefix' => ''])->get('QueueUrls');
        foreach ($queues as $queue) {
            $tokens = explode('/', $queue);
            $queueNames[] = $tokens[sizeof($tokens) - 1];
        }
        return $queueNames;
    }
480654
public function getNextUrl() {
        $urls = $this->_readUrlQueue();
        $nextUrl = array_shift($urls);
        $this->_writeUrlQueue($urls);
        return $nextUrl;
    }
1234943
protected function getCache()
    {
        $cache = $this->getCacheDriver();

        return $this->cacheTags ? $cache->tags($this->cacheTags) : $cache;
    }
1121353
protected function getCaches()
    {
        foreach ($this->caches as $key => $cache) {
            if (!$cache instanceof FlushableInterface) {
                $cache = $this->application->make($cache);
            }

            if ($cache instanceof FlushableInterface) {
                yield $key => $cache;
            }
        }
    }
155707
protected function getQueue()
    {
        if ($this->queue === null) {
            $queue = new \Expose\Queue\Mongo();
            $this->queue = $queue;
        } else {
            $queue = $this->queue;
        }
        return $queue;
    }
1042878
public function crawl( $url ) {

		$this->parse($url);

		while( $url = array_shift($this->sitemaps) ) {

			$this->parse($url);
		}

		return $this->urls;
	}
472073
public static function getUrls()
    {
        $urls = [];

        foreach (static::$_crumbs as $crumb) {
            if (!empty($crumb['url'])) {
                $urls[] = $crumb['url'];
            } elseif (is_object($crumb)) {
                $urls[] = $crumb->url;
            }
        }

        return $urls;
    }