Starts crawling the URI provided on instantiation

@return void
558383
public function crawl()
    {
        $this->getQueueManager()->addUri($this->seed);
        $this->getDownloader()->getPersistenceHandler()->setSpiderId($this->spiderId);

        $this->doCrawl();
    }
1095796
public static function init(string $uri = null): void
    {
        if (!defined('KOLDY_CLI')) {
            // KOLDY_CLI is not defined, which probably means useConfig() wasn't called
            static::terminateWithError('Don\'t call \Koldy\Application::run() before calling useConfig(). Check the documentation');
        }

        spl_autoload_register('\Koldy\Application::autoload');

        // set the error reporting in development mode
        if (static::inDevelopment()) {
            error_reporting(-1);
        }

        $config = static::getConfig('application');

        date_default_timezone_set($config->get('timezone', 'UTC'));

        $includePaths = [];
        $basePath = static::getApplicationPath();

        // register include path of application itself
        $includePaths[] = $basePath . 'controllers';

        if (is_dir($basePath . 'library')) {
            $includePaths[] = $basePath . 'library';
        }

        // adding additional include paths if there are any
        $additionalIncludePath = $config->get('additional_include_path') ?? [];

        // set the include path
        static::addIncludePath($includePaths, $additionalIncludePath);

        // set the error handler
        set_error_handler(function ($errorNumber, $errorMessage, $errorFile, $errorLine) {
            if (!(error_reporting() & $errorNumber)) {
                // This error code is not included in error_reporting
                return;
            }

            switch ($errorNumber) {
                case E_USER_ERROR:
                    $logMessage = new \Koldy\Log\Message('error');
                    $logMessage->addPHPErrorMessage($errorMessage, $errorFile, $errorNumber, $errorLine);
                    Log::message($logMessage);
                    break;

                case E_USER_WARNING:
                case E_DEPRECATED:
                case E_STRICT:
                    $logMessage = new \Koldy\Log\Message('warning');
                    $logMessage->addPHPErrorMessage($errorMessage, $errorFile, $errorNumber, $errorLine);
                    Log::message($logMessage);
                    break;

                case E_USER_NOTICE:
                    $logMessage = new \Koldy\Log\Message('notice');
                    $logMessage->addPHPErrorMessage($errorMessage, $errorFile, $errorNumber, $errorLine);
                    Log::message($logMessage);
                    break;


                default:
                    $logMessage = new \Koldy\Log\Message('warning');
                    $logMessage->addPHPErrorMessage($errorMessage, $errorFile, $errorNumber, $errorLine);
                    Log::message($logMessage);
                    break;
            }

            /* Don't execute PHP internal error handler */
            return true;
        });

        // register PHP fatal errors
        register_shutdown_function(function () {
            if (!defined('KOLDY_FATAL_ERROR_HANDLER')) {
                // to prevent possible recursion if you run into problems with logger
                define('KOLDY_FATAL_ERROR_HANDLER', true);

                $fatalError = error_get_last();

                if ($fatalError !== null && $fatalError['type'] == E_ERROR) {
                    $errorNumber = E_ERROR;
                    $errorMessage = $fatalError['message'];
                    $errorFile = $fatalError['file'];
                    $errorLine = $fatalError['line'];

                    $logMessage = new \Koldy\Log\Message('error');
                    $logMessage->addPHPErrorMessage($errorMessage, $errorFile, $errorNumber, $errorLine);

                    Log::error($logMessage);
                }
            }
        });

        // check the log for any enabled adapter
        $logConfigs = static::getConfig('application')->get('log', []);
        $logConfigsCount = count($logConfigs);
        $logEnabled = false;

        for ($i = 0; $i < $logConfigsCount && !$logEnabled; $i++) {
            if (isset($logConfigs[$i]['enabled']) && $logConfigs[$i]['enabled'] === true) {
                $logEnabled = true;
            }
        }

        if ($logEnabled) {
            // TODO: Don't init if we don't have to and let's do the "post" work after request
            Log::init();
        }
    }
943541
public function Init($url) {
        $urlParts = explode("/", $url);

        //@todo: Get the application base url
        $baseUrl = "/";
        $baseUrlConstainsVirtualPath = !(strcasecmp("/", $baseUrl) === 0);
        $startIndex = $baseUrlConstainsVirtualPath ? self::StartIndexWithVirtualPath : self::StartIndexNoVirtualPath;

        $this->setUrl($url);
        if (array_key_exists($startIndex, $urlParts) && array_key_exists($startIndex + 1, $urlParts)) {
            $this->setModule($urlParts[$startIndex]);
            $this->setAction($urlParts[$startIndex + 1]);
        } else {
            $this->Init($this->defaultUrl);
        }
    }
287111
private function parseRequestUri(): void
    {
        $requestUri = $this->baseClass->getRequestUri();
        $path = \explode('/', $requestUri);
        $pathCnt = \count($path);
        if ($pathCnt > 0) {
            $file = \array_pop($path);
            $path = \sprintf('%%DOCUMENT_ROOT%%%s%s', \implode(DIRECTORY_SEPARATOR, $path), DIRECTORY_SEPARATOR);
            $path = $this->baseClass->parsePath($path);

            $file = \explode('.', $file);
            $ext = '';

            if (\count($file) > 1) {
                $ext = \array_pop($file);
            }

            $file = \explode('__', \implode('.', $file));
            $resizeParams = [];

            if (\count($file) > 1) {
                $resizeParams = $this->parseResizeParams(\explode('_', $file[1]));
            }

            $file = $file[0];

            $this->uriParams['path'] = $path;
            $this->uriParams['file'] = $file;
            $this->uriParams['extension'] = $ext;
            $this->uriParams['resize-params'] = $resizeParams;
        }
    }
218750
public function init()
    {
        $pages = $this->getModel()->getPages();

        // Check datas
        if (empty($pages)) {
            return;
        }

        global $admin_page_hooks;

        // Set the internal pointer of an array to its first element
        reset($pages);

        // Update identifier
        $identifier = Helpers::urlize(key($pages));
        $this->getModel()->setIdentifier($identifier);

        // Add root single menu if identifier is unknown
        if (!isset($admin_page_hooks[$identifier])) {
            $this->addRootPage();
        }

        // Iterate on pages
        foreach ($pages as $slug => $options) {
            if (empty($options)) {
                continue;
            }

            // Get slug
            $slug = Helpers::urlize($slug);

            // Add child menu
            $this->addChild($slug, $options);
        }
    }
958662
public function run()
    {
        try
        {
            $this['triggers']->callTrigger('before-match-request', $this);

            $this['request']->attributes->add(
                $this['router']->matchRequest($this['request'])
            );

            $this['triggers']->callTrigger('before-controller-resolver', $this);

            $this['response'] = $this->controllerResolver();
        }
        catch (ResourceNotFoundException $e)
        {
            $this['response'] = (new Controller($this))->serve404();
        }
        catch (\Exception $e)
        {
            $this['response'] = new Response();
            $this['response']->headers->set('Content-Type', 'text/plain');
            $this['response']->setStatusCode(Response::HTTP_INTERNAL_SERVER_ERROR);
            $this['response']->setContent($e->getMessage());
        }

        if ($this['x-frame-options']) {
            $this['response']->headers->set('x-frame-options', $this['x-frame-options']);
        }

        $this['response']->prepare($this['request']);

        $this['triggers']->callTrigger('before-send-response', $this);

        $this['response']->send();
    }
1159319
public function indexAction() {
        $request = $this->getRequest();
        if (!$request->getParam('id') || !$request->getParam('chunk')) {
            throw new Exception('Not enough parameters: "id" and "chunk" are required.');
        }
        $bb = $this->_initBrowsebox($request);
        $this->view->bb = $bb;
        $this->_helper->layout->setLayout('blank');
    }
176669
def crawler(site, uri=None):
    """"""
    config = load_site_config(site)
    model = _get_model('crawl', config, uri)
    visited_set, visited_uri_set, consume_set, crawl_set = get_site_sets(
        site, config
    )

    if not visited_set.has(model.hash):
        visited_set.add(model.hash)
        visited_uri_set.add(model.uri)

        if (
            model.is_consume_page
            and not consume_set.has(model.hash)
        ):
            consume_set.add(model.hash)
            consume_q.enqueue(
                consumer,
                site,
                model.uri
            )
        else:
            for crawl_uri in model.uris_to_crawl:
                if (
                    not visited_uri_set.has(crawl_uri)
                    and not crawl_set.has(crawl_uri)
                ):
                    crawl_set.add(crawl_uri)
                    crawl_q.enqueue(
                        crawler,
                        site,
                        crawl_uri
                    )
1250523
public function routeHttp()
    {
        if ($this->config->isInitialized() || $this->url->isInstall()) {
            $this->route->process();
        } else {
            $this->url->redirect('install');
        }
    }
165398
public static function dispatch()
    {
        $uri = strtok($_SERVER["REQUEST_URI"],'?');
        $qs = parse_url($_SERVER["REQUEST_URI"], PHP_URL_QUERY);
        $method = $_SERVER['REQUEST_METHOD'];
        
        
        foreach (self::$routes as $name=>$route) {


            $route  = self::parsePattern($route);

            $matched = array();
            if ($route=='' || preg_match('#' . $route . '#', $uri, $matched) && (self::$methods[$name] == 'ANY' || self::$methods[$name] == $method)) {
                
                array_shift($matched);

                if (self::$qs[$name]!='' && preg_match('#' . self::$qs[$name] . '#', $qs, $qsmatched)) {

                    array_shift($qsmatched);

                    $matched = array_merge($matched, $qsmatched);
                    self::$catched[] = $name;
                    BurpEvent::listen($name, self::$callbacks[$name]);
                    BurpEvent::queue($name, $matched);

                } elseif (self::$qs[$name] == '') {
                    self::$catched[] = $name;
                    BurpEvent::listen($name, self::$callbacks[$name]);
                    BurpEvent::queue($name, $matched);
                }
            }

        }

        //call missing if needed
        if (!is_null(self::$missing_callback) ) {
            $diff = array_diff(self::$tocatch, self::$catched);
            if (count($diff) >= count(self::$tocatch)) {
                call_user_func(self::$missing_callback);
                BurpEvent::listen('missing', self::$callbacks[$name]);
                BurpEvent::fire('missing');

            }

        }

        BurpEvent::flushAll();
    }
621585
public static function run()
    {
        self::init();

        /*
         * Register routes.
         */
        call_user_func(function () {
            /** @noinspection PhpIncludeInspection */
            require self::$basePath . '/boot/routes.php';
            if (file_exists(self::$basePath . '/.manifest/plugins/routes.php')) {
                /** @noinspection PhpIncludeInspection */
                include self::$basePath . '/.manifest/plugins/routes.php';
            }
        });

        /*
         * Dispatch the HTTP request and send the response to the browser.
         */
        /** @var \Core\Services\Contracts\Router $router */
        $router   = DI::getInstance()->get('router');
        $request  = DI::getInstance()->get('request');
        $response = $router->dispatch($request);
        $response->send();
    }
1042799
public function startIndexingAction()
    {
        // make indexer instance and init
        /* @var $indexer IndexerRunner */
        $indexer = GeneralUtility::makeInstance(IndexerRunner::class);

        // get indexer configurations
        $indexerConfigurations = $indexer->getConfigurations();

        // get uri builder
        $uriBuilder = GeneralUtility::makeInstance(UriBuilder::class);

        $content = '';

        // action: start indexer or remove lock
        if ($this->do == 'startindexer') {
            // start indexing in verbose mode with cleanup process
            $content .= $indexer->startIndexing(true, $this->extConf);
        } else {
            if ($this->do == 'rmLock') {
                // remove lock from registry - admin only!
                if ($this->getBackendUser()->isAdmin()) {
                    $this->registry->removeAllByNamespace('tx_kesearch');
                } else {
                    $content .=
                        '<p>'
                        . LocalizationUtility::translate(
                            'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xml:not_allowed_remove_indexer_lock',
                            'KeSearch'
                        )
                        . '</p>';
                }
            }
        }


        // check for index process lock in registry
        // remove lock if older than 12 hours
        $lockTime = $this->registry->get('tx_kesearch', 'startTimeOfIndexer');
        $compareTime = time() - (60 * 60 * 12);
        if ($lockTime !== null && $lockTime < $compareTime) {
            // lock is older than 12 hours
            // remove lock and show "start index" button
            $this->registry->removeAllByNamespace('tx_kesearch');
            $lockTime = null;
        }

        // show information about indexer configurations and number of records
        // if action "start indexing" is not selected
        if ($this->do != 'startindexer') {
            $content .= $this->printNumberOfRecords();
            $content .= $this->printIndexerConfigurations($indexerConfigurations);
        }

        // show "start indexing" or "remove lock" button
        if ($lockTime !== null) {
            if (!$this->getBackendUser()->isAdmin()) {
                // print warning message for non-admins
                $content .= '<br /><p style="color: red; font-weight: bold;">WARNING!</p>';
                $content .= '<p>The indexer is already running and can not be started twice.</p>';
            } else {
                // show 'remove lock' button for admins
                $content .= '<br /><p>The indexer is already running and can not be started twice.</p>';
                $content .= '<p>The indexing process was started at ' . strftime('%c', $lockTime) . '.</p>';
                $content .= '<p>You can remove the lock by clicking the following button.</p>';
                $moduleUrl = $uriBuilder->buildUriFromRoute(
                    'web_KeSearchBackendModule',
                    [
                        'id' => $this->id,
                        'do' => 'rmLock'
                    ]
                );
                $content .= '<br /><a class="lock-button" href="' . $moduleUrl . '">RemoveLock</a>';
            }
        } else {
            // no lock set - show "start indexer" link if indexer configurations have been found
            if ($indexerConfigurations) {
                $moduleUrl = $uriBuilder->buildUriFromRoute(
                    'web_KeSearchBackendModule',
                    [
                        'id' => $this->id,
                        'do' => 'startindexer'
                    ]
                );
                $content .= '<br /><a class="index-button" href="' . $moduleUrl . '">'
                    .
                    LocalizationUtility::translate(
                        'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xml:start_indexer',
                        'KeSearch'
                    )
                    . '</a>';
            } else {
                $content .=
                    '<div class="alert alert-info">'
                    .
                    LocalizationUtility::translate(
                        'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xml:no_indexer_configurations',
                        'KeSearch'
                    )
                    . '</div>';
            }
        }

        $this->view->assign('content', $content);
    }
355883
static function Init()
	{
		require_once 'verysimple/Phreeze/Controller.php';
		require_once 'verysimple/HTTP/RequestUtil.php';

		if (!self::$IS_INITIALIZED)
		{
			RequestUtil::NormalizeUrlRewrite();

			Controller::$SmartyViewPrefix = '';
			Controller::$DefaultRedirectMode = 'header';

			self::$IS_INITIALIZED = true;
		}

	}
602075
protected function initCurl()
    {
        if (empty($this->curl)) {
            $this->curl = curl_init();
            curl_setopt_array($this->curl, $this->curlOpts);
            $this->log->debug('cURL object initialized', $this->curlOpts);
        }
    }
942243
public function start()
  {
    $router = new RouterCollection();
    require_once 'routes/web.php';
    require_once __DIR__ . '/defaultRouters.php';
    $route = new Router($router);
    $route->math()->connect();
  }
583795
public function init()
    {
        $this->routes = $this->routes->getRouteCollections();

        $this->dispatcher = $this->createDispatcher();

        $this->currentURL = $this->request->getPathInfo();

        $this->dispatch();
    }
906539
public static function initRequest()
    {
        global $PPHP;

        $parsedURL = parse_url($_SERVER['REQUEST_URI']);
        self::$_PATH = dejoin('/', trim($parsedURL['path'], '/'));
        while (count(self::$_PATH) > 0 && self::$_PATH[0] === '') {
            array_shift(self::$_PATH);
        };

        // Eat up initial directory as language if it's 2 characters
        $lang = $PPHP['config']['global']['default_lang'];
        if (isset(self::$_PATH[0]) && strlen(self::$_PATH[0]) === 2) {
            $lang = strtolower(array_shift(self::$_PATH));
        };
        self::$_req['lang'] = $lang;
        self::$_req['default_lang'] = $PPHP['config']['global']['default_lang'];
        self::$_req['base'] = ($lang === $PPHP['config']['global']['default_lang'] ? '' : "/{$lang}");

        // Eat up initial directories as long as they contain request flags
        self::$_req['binary'] = false;
        while (count(self::$_PATH) > 0 && self::$_PATH[0][0] === '=') {
            $flag = strtolower(array_shift(self::$_PATH));
            if ($flag === '=bin') {
                self::$_req['binary'] = true;
            };
        };

        self::$_req['path'] = implode('/', self::$_PATH);
        self::$_req['query'] = (isset($parsedURL['query']) ? '?' . $parsedURL['query'] : '');
        self::$_req['host'] = $_SERVER['HTTP_HOST'];
        self::$_req['remote_addr'] = self::_remoteIP();
        self::$_req['ssl']
            = (
                (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off')
                ||
                (isset($_SERVER['SERVER_PORT']) && $_SERVER['SERVER_PORT'] == 443)
                ||
                (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https')
                ||
                (isset($_SERVER['HTTP_X_FORWARDED_SSL']) && $_SERVER['HTTP_X_FORWARDED_SSL'] === 'on')
            );
        self::$_req['get'] = $_GET;
        self::$_req['post'] = $_POST;

        foreach (array('get', 'post') as $method) {
            if (isset(self::$_req[$method]['__arrays'])) {
                foreach (self::$_req[$method]['__arrays'] as $key) {
                    if (!isset(self::$_req[$method][$key])) {
                        self::$_req[$method][$key] = array();
                    } elseif (!is_array(self::$_req[$method][$key])) {
                        self::$_req[$method][$key] = array(self::$_req[$method][$key]);
                    };
                };
            };
        };

        // Process file uploads
        self::$_req['files'] = array();
        if (isset($_FILES) && count($_FILES) > 0) {
            $finfo = finfo_open(FILEINFO_MIME_TYPE);
            foreach ($_FILES as $key => $info) {
                if (is_uploaded_file($info['tmp_name'])) {
                    self::$_req['files'][$key] = $info;
                    // Overwrite client's MIME Type with server's determination
                    self::$_req['files'][$key]['type'] = finfo_file($finfo, $info['tmp_name']);
                    foreach (pathinfo($info['name']) as $pik => $piv) {
                        self::$_req['files'][$key][$pik] = $piv;
                    };
                };
            };
            finfo_close($finfo);
        };

        self::_initCommon();
    }