Feeding the spider
330004
function addUrl(when, feed) {
  if (!types.check(feed, 'feed') && !types.check(feed, ['feed']))
    throw Error('sandcrawler.spider.url(s): wrong argument.');

  var a = !(feed instanceof Array) ? [feed] : feed,
      c = !this.state.running ? this.initialBuffer.length : this.index,
      job,
      i,
      l;

  for (i = 0, l = a.length; i < l; i++) {
    job = createJob(a[i]);

    // Don't add if already at limit
    if (this.options.limit && c >= this.options.limit)
      break;

    if (!this.state.running) {
      this.initialBuffer[when === 'later' ? 'push' : 'unshift'](job);
    }
    else {
      this.queue[when === 'later' ? 'push' : 'unshift'](job);
      this.emit('job:add', job);
    }
  }

  return this;
}
209687
@Override
    protected void doPerform() {
        WebElement targetElem = getFirstElement(target);
        if (targetOffset == null) {
            getActions().dragAndDrop(getFirstElement(), targetElem).perform();
        } else {
            org.openqa.selenium.Dimension size = targetElem.getSize();
            Dimension dimension = new Dimension(size.width, size.height);
            Point offsetPoint = targetOffset.offset(dimension);
            getActions().dragAndDropBy(getFirstElement(), offsetPoint.x(), offsetPoint.y()).perform();
        }
    }
331892
def parse(self, response):
        
        yield scrapy.Request(self.helper.url_extractor.get_rss_url(response),
                             callback=self.rss_parse)
714884
def get_spider_stats(self, spider_name):
        """"""
        if spider_name is None:
            spider_name = self.spider_name
        else:
            self.spider_name = spider_name
        if self.spider_name is None:
            self.spider_name = self.list_running()[0].split(':')[-1]
        return(self.jsonrpc_call('stats', 'get_stats', self.spider_name))
742666
def feed(self, contents):
        '''
            
        '''
        contents = stripIEConditionals(contents)
        try:
            HTMLParser.feed(self, contents)
        except MultipleRootNodeException:
            self.reset()
            HTMLParser.feed(self, "%s%s" %(addStartTag(contents, INVISIBLE_ROOT_TAG_START), INVISIBLE_ROOT_TAG_END))
338075
def next_feed(feed = nil)
      return Feed.load_xml(get_collection(collection_uri).body) if feed.nil?
      return nil unless feed.has_next?
      Feed.load_xml(get_collection(feed.next_uri).body)
    end
45316
def feed(self, data):
        
        """
        self.rawdata = self.rawdata + data
        self.goahead(0)
1043553
private void complete() {
		if (stopped) {
			return;
		}

		log.info("Spidering process is complete. Shutting down...");
		this.stopped = true;
		if (httpSender != null) {
			this.getHttpSender().shutdown();
			httpSender = null;
		}

		// Notify the controller to clean up memory
		controller.reset();

		// Issue the shutdown command on a separate thread, as the current thread is most likely one
		// from the pool
		new Thread(new Runnable() {
			@Override
			public void run() {
				if (threadPool != null) {
					threadPool.shutdown();
				}
				// Notify the listeners -- in the meanwhile
				notifyListenersSpiderComplete(true);
				controller.reset();
				threadPool = null;
			}
		}, "ZAP-SpiderShutdownThread-" + id).start();
	}
732745
def handle(self, key, value):
        '''
        
        '''
        # break down key
        elements = key.split(":")

        if len(elements) != 4:
            self.logger.warn("Stop requests need a crawlid and appid")
            return

        spiderid = elements[1]
        appid = elements[2]
        crawlid = elements[3]
        uuid = value

        # log we received the stop message
        extras = self.get_log_dict('stop', appid,
                                   spiderid, uuid, crawlid)
        self.logger.info('Received stop request', extra=extras)

        redis_key = spiderid + ":blacklist"
        value = '{appid}||{crawlid}'.format(appid=appid,
                                            crawlid=crawlid)

        # add this to the blacklist set
        self.redis_conn.sadd(redis_key, value)

        # purge crawlid from current set
        result = self._purge_crawl(spiderid, appid, crawlid)

        # item to send to kafka
        extras = {}
        extras['action'] = "stop"
        extras['spiderid'] = spiderid
        extras['appid'] = appid
        extras['crawlid'] = crawlid
        extras['total_purged'] = result
        extras['uuid'] = uuid
        extras['server_time'] = int(self.get_current_time())

        if self._send_to_kafka(extras):
            # delete timeout for crawl (if needed) since stopped
            timeout_key = 'timeout:{sid}:{aid}:{cid}'.format(
                                    sid=spiderid,
                                    aid=appid,
                                    cid=crawlid)
            self.redis_conn.delete(timeout_key)
            extras['success'] = True
            self.logger.info('Sent stop ack to kafka', extra=extras)
        else:
            extras['success'] = False
            self.logger.error('Failed to send stop ack to kafka', extra=extras)
908363
def schedule_crawl(spider, workflow, **kwargs):
    """"""
    from inspire_crawler.utils import get_crawler_instance

    crawler = get_crawler_instance()
    crawler_settings = current_app.config.get('CRAWLER_SETTINGS')
    crawler_settings.update(kwargs.get("crawler_settings", {}))

    crawler_arguments = kwargs
    crawler_arguments.update(
        current_app.config.get('CRAWLER_SPIDER_ARGUMENTS', {}).get(spider, {})
    )
    job_id = crawler.schedule(
        project=current_app.config.get('CRAWLER_PROJECT'),
        spider=spider,
        settings=crawler_settings,
        **crawler_arguments
    )
    if job_id:
        crawler_job = CrawlerJob.create(
            job_id=job_id,
            spider=spider,
            workflow=workflow,
        )
        db.session.commit()
        current_app.logger.info(
            "Scheduled scrapyd job with id: {0}".format(job_id)
        )
        current_app.logger.info(
            "Created crawler job with id:{0}".format(crawler_job.id)
        )
    else:
        raise CrawlerScheduleError(
            "Could not schedule '{0}' spider for project '{1}'".format(
                spider, current_app.config.get('CRAWLER_PROJECT')
            )
        )

    return str(crawler_job.job_id)
1008163
def in_feed(self, feed):
        ''''''
        kwargs = {self.model._rel_to_feed: feed}
        return self.filter(**kwargs)
241829
def feed(self, data):
        
        send = self._send_to_parser
        draw = self.listener.draw
        match_text = self._text_pattern.match
        taking_plain_text = self._taking_plain_text

        length = len(data)
        offset = 0
        while offset < length:
            if taking_plain_text:
                match = match_text(data, offset)
                if match:
                    start, offset = match.span()
                    draw(data[start:offset])
                else:
                    taking_plain_text = False
            else:
                taking_plain_text = send(data[offset:offset + 1])
                offset += 1

        self._taking_plain_text = taking_plain_text
1188363
def manage_crawler(self):
        
        index = True
        self.number_of_active_crawlers += 1
        while not self.shutdown and index is not None:
            index = self.crawler_list.get_next_item()
            if index is None:
                self.number_of_active_crawlers -= 1
                break

            self.start_crawler(index)
626169
def crawl(self):
        """"""
        page = Linkfetcher(self.root)
        page.linkfetch()
        queue = Queue()
        for url in page.urls:
            queue.put(url)
        followed = [self.root]

        n = 0

        while True:
            try:
                url = queue.get()
            except QueueEmpty:
                break

            n += 1

            if url not in followed:
                try:

                    host = urllib.parse.urlparse(url)[1]

                    if self.locked and re.match(".*%s" % self.host, host):
                        followed.append(url)
                        self.followed += 1
                        page = Linkfetcher(url)
                        page.linkfetch()
                        for i, url in enumerate(page):
                            if url not in self.urls:
                                self.links += 1
                                queue.put(url)
                                self.urls.append(url)

                        if n > self.depth and self.depth > 0:
                            break
                except Exception as e:
                    print("ERROR: The URL '%s' can't be crawled (%s)" % (url, e))
                    print(format_exc())
554703
def feed(self, data):
        
        self.buffer += data
        while len(self.buffer) >= 6:
            self.next_packet()
738828
def process_item(self, item, spider):  # pylint: disable=unused-argument
        
        url = self.clean_url(item["url"])

        if self.is_sequence_start_page(url):
            url = url.parent

        if url in self.urls_seen:
            raise DropItem(u"Dropping duplicate url {url}".format(url=item["url"]))
        else:
            self.urls_seen.add(url)
            return item
1008164
function crawlImage ($,result, img, callback) {
      var parentUrl = result.url;

      var link = $(img).attr('src');
      var alt = $(img).attr('alt');
      if (link) {
          var linkUri = URI.linkToURI(parentUrl, link);
          pm.crawlImage(parentUrl, linkUri, alt, function(){
            addLinkToQueue(result, parentUrl, linkUri, null, null, callback);
          });
      }
      else {
        callback();
      }
}