Run crawlers that are due.
427130
def scheduled(wait=False):
    """"""
    manager.run_scheduled()
    while wait:
        # Loop and try to run scheduled crawlers at short intervals
        manager.run_scheduled()
        time.sleep(settings.SCHEDULER_INTERVAL)
1077589
public function finishedCrawling()
    {
        $this->log->info('link checker summary');

        collect($this->urlsGroupedByStatusCode)
            ->each(function ($urls, $statusCode) {
                if ($this->isSuccessOrRedirect($statusCode)) {
                    return;
                }

                $count = count($urls);

                if ($statusCode == static::UNRESPONSIVE_HOST) {
                    $this->log->warning("{$count} url(s) did have unresponsive host(s)");

                    return;
                }

                $this->log->warning("Crawled {$count} url(s) with statuscode {$statusCode}");
            });
    }
1239408
def run(*extractor_list, **kwargs):
    """"""
    args = _get_args(kwargs.get('args'))
    n_extractors = len(extractor_list)
    log.info('Going to run list of {} FeatureExtractors'.format(n_extractors))
    collection = fex.Collection(cache_path=args.cache_path)
    for extractor in extractor_list:
        collection.add_feature_extractor(extractor)

    out_path = args.path
    if args.deploy:
        out_path = _prefix_git_hash(out_path)
    collection.run(out_path)
335652
private synchronized void maybeRun() {
        if (inprogress==null && pending!=null) {
            base.submit(new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    synchronized (AtmostOneTaskExecutor.this) {
                        // everyone who submits after this should form a next batch
                        inprogress = pending;
                        pending = null;
                    }

                    try {
                        inprogress.set(task.call());
                    } catch (Throwable t) {
                        LOGGER.log(Level.WARNING, null, t);
                        inprogress.setException(t);
                    } finally {
                        synchronized (AtmostOneTaskExecutor.this) {
                            // if next one is pending, get that scheduled
                            inprogress = null;
                            maybeRun();
                        }
                    }
                    return null;
                }
            });
        }
    }
1303987
private void performDelay() {
        try {
            TimeUnit.MILLISECONDS.sleep(crawlDelayMechanism.getDelay());
        } catch (InterruptedException ex) {
            Thread.currentThread().interrupt();
            isStopping = true;
        }
    }
170703
def run(self):
        """"""
        try:
            while not self.stopped:
                scraperobj = jobs.get(False)
                self.setName(scraperobj.getName())
                try:
                    self.getStrips(scraperobj)
                finally:
                    jobs.task_done()
                    self.setName(self.origname)
        except Empty:
            pass
        except KeyboardInterrupt:
            thread.interrupt_main()
1037458
def run(self):
        """"""

        idle_watchers = []

        while True:
            try:
                if self._work_queue.empty() and len(idle_watchers) > 0:
                    for watcher in idle_watchers:
                        try:
                            watcher()
                        except: # pylint:disable=bare-except;We can't let one idle watcher failure impact any other watcher
                            self._logger.exception("Error inside queue idle watcher")

                    idle_watchers = []

                item = self._work_queue.get()

                # Handle special actions that are not RPCs
                if item is STOP_WORKER_ITEM:
                    return
                elif isinstance(item, MarkLocationItem):
                    item.callback()
                    continue
                elif isinstance(item, WaitIdleItem):
                    idle_watchers.append(item.callback)
                    continue
                elif not isinstance(item, WorkItem):
                    self._logger.error("Invalid item passed to WorkQueueThread: %s, ignoring", item)
                    continue

                self.direct_dispatch(item.arg, item.callback)

            except:  # pylint:disable=bare-except;We cannot let this background thread die until we are told to stop()
                self._logger.exception("Error inside background workqueue thread")
609450
def run(self):
        ''''''
        for line in iter(self._fd.readline, ''):
            self._queue.put(line)
1187604
protected final void crawl(final CrawlRequest request) {
        Validate.validState(!isStopped,
                "The crawler is not started. Maybe you meant to add this request as a crawl seed?");
        Validate.validState(!isStopping, "Cannot add request when the crawler is stopping.");
        Validate.notNull(request, "The request cannot be null.");

        crawlFrontier.feedRequest(request, false);
    }
1229218
def run(self):
        """"""
        # Create the thread pool.
        executor = concurrent.futures.ThreadPoolExecutor(
            max_workers=self._config['num_workers'])

        # Wait to ensure multiple senders can be synchronised.
        now = int(datetime.datetime.utcnow().timestamp())
        start_time = ((now + 29) // 30) * 30
        self._log.info('Waiting until {}'.format(
                       datetime.datetime.fromtimestamp(start_time)))
        while int(datetime.datetime.utcnow().timestamp()) < start_time:
            time.sleep(0.1)

        # Run the event loop.
        loop = asyncio.get_event_loop()
        try:
            loop.run_until_complete(self._run_loop(executor))
        except KeyboardInterrupt:
            pass
        finally:
            # Send the end of stream message to each stream.
            self._log.info('Shutting down, closing streams...')
            tasks = []
            for stream, item_group in self._streams:
                tasks.append(stream.async_send_heap(item_group.get_end()))
            loop.run_until_complete(asyncio.gather(*tasks))
            self._log.info('... finished.')
            executor.shutdown()
1330379
def execute(self, data):
        """"""
        if Crawl.is_aborted(self.crawler, self.run_id):
            return

        try:
            Crawl.operation_start(self.crawler, self.stage, self.run_id)
            self.log.info('[%s->%s(%s)]: %s',
                          self.crawler.name,
                          self.stage.name,
                          self.stage.method_name,
                          self.run_id)
            return self.stage.method(self, data)
        except Exception as exc:
            self.emit_exception(exc)
        finally:
            Crawl.operation_end(self.crawler, self.run_id)
            shutil.rmtree(self.work_path)
661265
public static void main(String[] args) throws IOException {
		CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(URL);
		
		builder.crawlRules().setFormFillMode(FormFillMode.RANDOM);

		// click these elements
		builder.crawlRules().clickDefaultElements();
		 /*builder.crawlRules().click("A");
		 builder.crawlRules().click("button");*/
		builder.crawlRules().crawlHiddenAnchors(true);
		builder.crawlRules().crawlFrames(false);
		builder.setUnlimitedCrawlDepth();
		builder.setUnlimitedRuntime();
		builder.setUnlimitedStates();

		//builder.setMaximumStates(10);
		//builder.setMaximumDepth(3);
		builder.crawlRules().clickElementsInRandomOrder(false);

		// Set timeouts
		builder.crawlRules().waitAfterReloadUrl(WAIT_TIME_AFTER_RELOAD, TimeUnit.MILLISECONDS);
		builder.crawlRules().waitAfterEvent(WAIT_TIME_AFTER_EVENT, TimeUnit.MILLISECONDS);

		builder.setBrowserConfig(new BrowserConfiguration(BrowserType.PHANTOMJS, 1));

		// CrawlOverview
		builder.addPlugin(new CrawlOverview());

		CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
		crawljax.call();

	}
468149
public static void main(String[] args) {
		CrawljaxRunner crawljax =
		        new CrawljaxRunner(CrawljaxConfiguration.builderFor("http://demo.crawljax.com/")
		                .build());
		crawljax.call();
	}
166799
def run(self):
        """"""
        for wsock in self.wsocks:
            wsock.run()
        for api in self.apis:
            api.run()
391885
public function run()
    {
        $this->seed([
            [
                'group'       => [
                    'name'        => 'SEO',
                    'slug'        => 'seo',
                    'description' => 'SEO permissions group',
                ],
                'permissions' => array_merge(
                    $this->getDashboardSeeds(),
                    $this->getPagesSeeds(),
                    $this->getFootersSeeds(),
                    $this->getMetasSeeds(),
                    $this->getRedirectsSeeds()
                ),
            ],
        ]);
    }
1201720
def robots(self):
        """"""
        r = 'noindex' if self.is_noindex else 'index'
        r += ','
        r += 'nofollow' if self.is_nofollow else 'follow'
        return r
1327963
def _runner(self, classpath, main, jvm_options, args, cwd=None):
    """"""
    command = self._create_command(classpath, main, jvm_options, args)

    class Runner(self.Runner):
      @property
      def executor(this):
        return self

      @property
      def command(self):
        return list(command)

      def run(this, stdout=None, stderr=None, stdin=None, cwd=None):
        nailgun = None
        try:
          nailgun = self._get_nailgun_client(jvm_options, classpath, stdout, stderr, stdin)
          logger.debug('Executing via {ng_desc}: {cmd}'.format(ng_desc=nailgun, cmd=this.cmd))
          return nailgun.execute(main, cwd, *args)
        except (NailgunClient.NailgunError, self.InitialNailgunConnectTimedOut) as e:
          self.terminate()
          raise self.Error('Problem launching via {ng_desc} command {main} {args}: {msg}'
                           .format(ng_desc=nailgun or '<no nailgun connection>',
                                   main=main,
                                   args=' '.join(args),
                                   msg=e))

    return Runner()