Starts a crawler from the input-array.

        :param int index: The array-index of the site
        :param int daemonize: Bool if the crawler is supposed to be daemonized
                              (to delete the JOBDIR)
190471
def start_crawler(self, index, daemonize=False):
        
        call_process = [sys.executable,
                        self.__single_crawler,
                        self.cfg_file_path,
                        self.json_file_path,
                        "%s" % index,
                        "%s" % self.shall_resume,
                        "%s" % daemonize]

        self.log.debug("Calling Process: %s", call_process)

        crawler = Popen(call_process,
                        stderr=None,
                        stdout=None)
        crawler.communicate()
        self.crawlers.append(crawler)
1323864
def _delete_from_indices(self, pipeline):
        
        s = Set(self.key()['_indices'])
        z = Set(self.key()['_zindices'])
        for index in s.members:
            pipeline.srem(index, self.id)
        for index in z.members:
            pipeline.zrem(index, self.id)
        pipeline.delete(s.key)
        pipeline.delete(z.key)
558383
public function crawl()
    {
        $this->getQueueManager()->addUri($this->seed);
        $this->getDownloader()->getPersistenceHandler()->setSpiderId($this->spiderId);

        $this->doCrawl();
    }
824582
def remove_jobdir_if_not_resume(self):
        
        jobdir = self.__scrapy_options["JOBDIR"]

        if (not self.shall_resume or self.daemonize) \
                and os.path.exists(jobdir):
            shutil.rmtree(jobdir)

            self.log.info("Removed " + jobdir + " since '--resume' was not passed to"
                                                " initial.py or this crawler was daemonized.")
236344
def __request_start(self, queue_item):
        

        try:
            action = self.__options.callbacks.request_before_start(self.queue, queue_item)
        except Exception as e:
            action = None
            print(e)
            print(traceback.format_exc())

        if action == CrawlerActions.DO_STOP_CRAWLING:
            self.__should_stop = True

        if action == CrawlerActions.DO_SKIP_TO_NEXT:
            self.queue.move(queue_item, QueueItem.STATUS_FINISHED)
            self.__should_spawn_new_requests = True

        if action == CrawlerActions.DO_CONTINUE_CRAWLING or action is None:
            self.queue.move(queue_item, QueueItem.STATUS_IN_PROGRESS)

            thread = CrawlerThread(self.__request_finish, self.__lock, self.__options, queue_item)
            self.__threads[queue_item.get_hash()] = thread
            thread.daemon = True
            thread.start()
1327439
def skip( index, start )
      if @index == index
        Lookup.new @p.lookup( start, @stride ), @index, @stride
      else
        Lookup.new @p.skip( index, start ), @index, @stride
      end
    end
43147
def start_index(self, value):
        """"""
        # TODO: Validate contents?  (May want to set before adding the data.)
        if not isinstance(value, dict):
            raise TypeError('start_index attribute must be a dict.')
        self._start_index = value
656566
def create_index(index):
    """"""
    logger.info("Creating search index: '%s'", index)
    client = get_client()
    return client.indices.create(index=index, body=get_index_mapping(index))
449506
public static void initializeIndex(String indexDir, String dataDir) {
        File indexDir_f = new File(indexDir);
        File dataDir_f = new File(dataDir);

        long start = new Date().getTime();
        try {
            int numIndexed = index(indexDir_f, dataDir_f);
            long end = new Date().getTime();

            System.err.println("Indexing " + numIndexed + " files took " + (end -start) + " milliseconds");
        } catch (IOException e) {
            System.err.println("Unable to index "+indexDir_f+": "+e.getMessage());
        }
    }
70666
def __start(self):
        
        thread = Thread(target=self.__loop, args=())
        thread.daemon = True  # daemonize thread
        thread.start()
        self.__enabled = True
830872
def dict_crawl(entry, key):
    ''' 
    '''
    value = None
    key_list = make_list(key)
    try:
        result = None
        success_flag = True
        temp = entry
        for next_key in key_list:
            parent = temp
            actual_type = detect_type(temp)
            if actual_type=="mongoengine":
                if next_key in temp.__dict__['_data']:
                    temp = temp.__dict__['_data'][next_key]
                else:
                    break
            elif actual_type=="class":
                if next_key in temp.__dict__:
                    temp = temp.__dict__[next_key]
                else:
                    break
            elif actual_type=="dict":
                if next_key in temp:
                    temp = temp[next_key]
                else:
                    break
            elif actual_type=="iterable_dict":
                if next_key in convert_to_dict(temp): # 'contains' method might not be there
                    temp = temp[next_key]
                else:
                    break
            elif actual_type=="list":
                if next_key in temp:
                    temp = temp[next_key]
                else:
                    break
            else:  ## I'm unsure we can really 'crawl' an 'object'
                break
        else:
            return (parent, key_list[-1], temp)
        return (None, None, None)
    except:
        pass
    return (None, None, None)
64677
def delete_index(es, index_name: str):
    """"""

    if not index_name:
        log.warn("No index name given to delete")
        return None

    result = es.indices.delete(index=index_name)
    return result
1143381
def _delIndex(catalog, index):
    
    if index in catalog.indexes():
        try:
            catalog.delIndex(index)
            logger.info(
                'Catalog index %s deleted from %s.' % (index, catalog.id))
            return True
        except:
            logger.error(
                'Catalog index %s error while deleting from %s.'
                % (index, catalog.id))
    return False
245710
def run(self):
        
        logger.info("Starting daemon.")
        while True:
            try:
                self._scan_disk()
                do_cleaning, delete_from_index = self._analyze_file_index()
                if do_cleaning:
                    self._clean_cache(delete_from_index)
            except Exception:
                logger.exception("Following exception occurred:")
            sleeping_until_time = datetime.datetime.now() + self.scan_interval
            logger.info("Sleeping until %s.", sleeping_until_time)
            time.sleep(self.scan_interval.total_seconds())
33221
def index(self, index):
        
        if self._index != index:
            self._dirty = True
        self._index = index
371018
def _purge_crawl(self, spiderid, appid, crawlid):
        '''
        
        '''
        # purge three times to try to make sure everything is cleaned
        total = self._mini_purge(spiderid, appid, crawlid)
        total = total + self._mini_purge(spiderid, appid, crawlid)
        total = total + self._mini_purge(spiderid, appid, crawlid)

        return total
992374
def start(self):
        
        # Check for a pidfile to see if the daemon already runs
        pid = None
        if os.path.exists(self.pidfile):
            try:
                pf = file(self.pidfile,'r')
                pid = int(pf.read().strip())
                pf.close()
            except IOError:
                pid = None

        if pid:
            message = "pidfile %s already exist. Daemon already running?\n"
            sys.stderr.write(message % self.pidfile)
            sys.exit(1)

        # 
        self.daemonize()
        self.run()