@param Crawler $crawler

@return false|string
964008
private function extractDetailUrlFromCrawler(Crawler $crawler)
    {
        $linksFound = $crawler->filter('.result-links > li > a');
        if (!$linksFound) {
            return false;
        }

        $href = $linksFound->extract(['href']);
        if (!isset($href[1])) {
            return false;
        }

        return self::URL_BASE.$href[1];
    }
1112305
function(crawler) {
		var dir = gui.Spirit.prototype.oncrawler.call(this, crawler);
		if (dir === gui.Crawler.CONTINUE) {
			switch (crawler.type) {
				case gui.CRAWLER_VISIBLE:
				case gui.CRAWLER_INVISIBLE:
					if (this._waiting) {
						dir = gui.Crawler.STOP;
					}
					break;
			}
		}
		return dir;
	}
844201
public function addCrawlers(array $crawlers)
    {
        foreach ($crawlers as $crawler) {
            if (!$crawler instanceof CrawlerInterface) {
                throw new \InvalidArgumentException('The $crawlers array may only contain FlameCore\Flink\Crawler\CrawlerInterface instances.');
            }

            $this->crawlers[] = $crawler;
        }
    }
1372432
public function getDescription(Crawler $node = null)
    {
        $node = $node ?: $this->node;

        return [
            'description' => $this->getFieldValue(
                $node->filter("p[itemprop=description]"),
                null
            ),
        ];
    }
467744
protected function getCurrentPageUrl($pager)
    {
        if ($pager->isFirstPage()) {
            $pageUrl = $pager->getPageUrl(null);
        } else {
            $pageUrl = $pager->getPageUrl($pager->getCurrentPage());
        }
        return $this->removeTags->filter($pageUrl);
    }
736990
public function matches($crawler): bool
    {
        $links = $this->crawler($crawler)->selectLink($this->text);

        if ($links->count() == 0) {
            return false;
        }

        // If the URL is null we assume the developer only wants to find a link
        // with the given text regardless of the URL. So if we find the link
        // we will return true. Otherwise, we will look for the given URL.
        if ($this->url == null) {
            return true;
        }

        $absoluteUrl = $this->absoluteUrl();

        foreach ($links as $link) {
            $linkHref = $link->getAttribute('href');

            if ($linkHref == $this->url || $linkHref == $absoluteUrl) {
                return true;
            }
        }

        return false;
    }
522330
private function scrapeUrl($url)
    {
        if (!parse_url($url, PHP_URL_HOST)) {
            $url = 'http://' . $url;
        }

        try {
            $client = new Client();
            $response = $client->get($url);
        } catch(\Exception $e) {
            return [];
        }

        $crawler = new Crawler($response->getBody()->getContents());

        $description = '';
        foreach ($crawler->filter('meta') as $node) {
            if ($node->getAttribute('name') == 'description') {
                $description = $node->getAttribute('content');
            }
        }

        $title = '';
        if (count($crawler->filter('title'))) {
            $title = trim($crawler->filter('title')->text());
        }

        return [
            'title' => $title,
            'description' => $description,
        ];
    }
1042878
public function crawl( $url ) {

		$this->parse($url);

		while( $url = array_shift($this->sitemaps) ) {

			$this->parse($url);
		}

		return $this->urls;
	}
824947
public function getProperties(Crawler $node = null)
    {
        $node = $node ?: $this->node;

        $properties = [
            'titre'      => (new DefaultSanitizer)->clean(
                $node->filter('h1')->text()
            ),
            'created_at' => $node
                ->filter('*[itemprop=availabilityStarts]')
                ->first()
                ->attr('content'),
            'is_pro'     => ($node->filter('.ispro')->count()),
        ];

        $node->filter('h2')
            ->each(function (Crawler $crawler) use (&$properties) {
                $properties = array_merge(
                    $properties,
                    $this->sanitize(
                        $crawler->filter('.property')->text(),
                        $crawler->filter('.value')->text()
                    )
                );
            });

        return ['properties' => $properties];
    }
1135993
protected function doFetch($url)
    {
        foreach ($this->crawlers as $crawler) {
            $data = $crawler->crawl($url);

            if (is_array($data)) {
                $title = isset($data['title']) ? $data['title'] : null;
                $description = isset($data['description']) ? $data['description'] : null;
                $type = isset($data['type']) ? $data['type'] : 'unknown';
                $images = isset($data['images']) ? (array) $data['images'] : array();

                return array(
                    'title'        => $title,
                    'description'  => $description,
                    'type'         => $type,
                    'images'       => $images,
                    'total_images' => count($images)
                );
            }
        }

        return false;
    }
79306
public static function removeChildNodes(Crawler $crawler): Crawler
    {
        if (!$crawler->count()) {
            return $crawler;
        }
        $crawler->children()->each(
            function (Crawler $crawler) {
                $node = $crawler->getNode(0);
                if ($node === null || $node->nodeType === 3 || \in_array($node->nodeName, self::ALLOWED_NODES, true)) {
                    return;
                }
                $node->parentNode->removeChild($node);
            }
        );

        return $crawler;
    }
1285037
private function parseShowtime(Crawler $timeSpan)
    {
        $showtime = new ShowtimeInfo();

        $texts = explode(' ', str_replace('&nbsp', '', $timeSpan->text()));

        if ($this->getTime($texts[0]) == null) {
            $showtime->setInfo($texts[0]);
        }

        $times = [];
        foreach ($texts as $text) {
            $time = trim(html_entity_decode($text));
            $time = $this->getTime($time);

            if (! empty($time)) {
                $times[] = $time;
            }
        }

        $showtime->setTimes(array_unique($times));

        return $showtime;
    }
413458
public function getInputOrTextAreaValue(Crawler $crawler)
    {
        $field = $this->field($crawler);

        return $field->nodeName() == 'input'
            ? $field->attr('value')
            : $field->text();
    }
1017115
public function getTitle()
    {
        $crawler = $this->getCrawler();

        $result = $crawler->filter('h1');
        if ($result->count()) {
            return $result->html();
        }
    }
182648
public function robot($userAgent = null)
    {
        if ($this->getCrawlerDetect()->isCrawler($userAgent ?: $this->userAgent)) {
            return ucfirst($this->getCrawlerDetect()->getMatches());
        }

        return false;
    }
965653
protected static function getReferrerDns($Referrer = false)
    {
        if ($Referrer === false)
        {
            $http_referrer = isset($_SERVER['HTTP_REFERER']) ? $_SERVER['HTTP_REFERER'] : 'unknown' ;
        }
        else
        {
            $http_referrer = $Referrer;
        }
        //nur den host Anteil prüfen
        $referrer_DNS = parse_url( $http_referrer, PHP_URL_HOST );
        if ($referrer_DNS === NULL)
        {
            //try this...
            $referrer_DNS = @parse_url( 'http://'.$http_referrer, PHP_URL_HOST );
            if ($referrer_DNS === NULL ||
                $referrer_DNS === false)
            {
                //wtf...
                return false;
            }
        }
        return $referrer_DNS;
    }
785814
public static function checkBrowserRobot()
    {
        if (stripos(self::$userAgentString, 'bot') !== false ||
            stripos(self::$userAgentString, 'spider') !== false ||
            stripos(self::$userAgentString, 'crawler') !== false
        ) {
            self::$browser->setIsRobot(true);

            return true;
        }

        return false;
    }