A decorator for setting the default kwargs of `BaseHandler.crawl`.
    Any self.crawl with this callback will use this config.
436178
def config(_config=None, **kwargs):
    
    if _config is None:
        _config = {}
    _config.update(kwargs)

    def wrapper(func):
        func._config = _config
        return func
    return wrapper
908363
def schedule_crawl(spider, workflow, **kwargs):
    """"""
    from inspire_crawler.utils import get_crawler_instance

    crawler = get_crawler_instance()
    crawler_settings = current_app.config.get('CRAWLER_SETTINGS')
    crawler_settings.update(kwargs.get("crawler_settings", {}))

    crawler_arguments = kwargs
    crawler_arguments.update(
        current_app.config.get('CRAWLER_SPIDER_ARGUMENTS', {}).get(spider, {})
    )
    job_id = crawler.schedule(
        project=current_app.config.get('CRAWLER_PROJECT'),
        spider=spider,
        settings=crawler_settings,
        **crawler_arguments
    )
    if job_id:
        crawler_job = CrawlerJob.create(
            job_id=job_id,
            spider=spider,
            workflow=workflow,
        )
        db.session.commit()
        current_app.logger.info(
            "Scheduled scrapyd job with id: {0}".format(job_id)
        )
        current_app.logger.info(
            "Created crawler job with id:{0}".format(crawler_job.id)
        )
    else:
        raise CrawlerScheduleError(
            "Could not schedule '{0}' spider for project '{1}'".format(
                spider, current_app.config.get('CRAWLER_PROJECT')
            )
        )

    return str(crawler_job.job_id)
444005
def _set(self, **kwargs):
        
        for param, value in kwargs.items():
            p = getattr(self, param)
            if value is not None:
                try:
                    value = p.typeConverter(value)
                except TypeError as e:
                    raise TypeError('Invalid param value given for param "%s". %s' % (p.name, e))
            self._paramMap[p] = value
        return self
643769
def prepare_request(self, **kwargs):
        

        if self.transport is None:
            self.setup_transport(self.transport_param)
        self.reset()
        self.request_counter = next(REQUEST_COUNTER)
        if kwargs:
            self.setup(**kwargs)
        if self.proxylist.size() and self.config['proxy_auto_change']:
            self.change_proxy()
        self.request_method = self.detect_request_method()
        self.transport.process_config(self)
487278
def setup_redis(self, crawler=None):
        
        if self.server is not None:
            return

        if crawler is None:
            # We allow optional crawler argument to keep backwards
            # compatibility.
            # XXX: Raise a deprecation warning.
            crawler = getattr(self, 'crawler', None)

        if crawler is None:
            raise ValueError("crawler is required")

        settings = crawler.settings

        if self.redis_key is None:
            self.redis_key = settings.get(
                'REDIS_START_URLS_KEY', defaults.START_URLS_KEY,
            )

        self.redis_key = self.redis_key % {'name': self.name}

        if not self.redis_key.strip():
            raise ValueError("redis_key must not be empty")

        if self.redis_batch_size is None:
            # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
            self.redis_batch_size = settings.getint(
                'REDIS_START_URLS_BATCH_SIZE',
                settings.getint('CONCURRENT_REQUESTS'),
            )

        try:
            self.redis_batch_size = int(self.redis_batch_size)
        except (TypeError, ValueError):
            raise ValueError("redis_batch_size must be an integer")

        if self.redis_encoding is None:
            self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)

        self.logger.info("Reading start URLs from redis key '%(redis_key)s' "
                         "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
                         self.__dict__)

        self.server = connection.from_settings(crawler.settings)
        # The idle signal is called when the spider has no requests left,
        # that's when we will schedule new requests from redis queue
        crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)
1147547
def args_wrapper(*args):
    
    out = defaultdict(list)
    for callback in args:
        callback_args = callback.callback_args()
        for k, v in callback_args.items():
            out[k].append(v)
    return dict(out)
1242688
def set_attributes(self, **kwargs):
        
        kwargs = dict(kwargs)
        for name,value in kwargs.items():
            # Raise AttributeError if param not found
            self.__getattr__(name) 
            # Set attributes
            try: self.setp(name,**value)
            except TypeError:
                try:  self.setp(name,*value)
                except (TypeError,KeyError):  
                    self.__setattr__(name,value)
162836
def kwargs(self):
        """"""
        # Subclasses must override this property if and only if they define
        # keyword-only arguments in their __init__ method
        if hasattr(self, '_has_kwargs') and self._has_kwargs:
            raise NotImplementedError(
                "Class %s does not provide a kwargs property"
                % str(self.__class__.__name__))
        return {}
1115965
@Override
  protected void configure() {
    super.configure();
    bind(Crawler.class).to(DefaultCrawler.class);
    bind(ExecutorService.class).toProvider(ExecutorServiceProvider.class);
    bind(HTMLPageResponseFetcher.class).to(HTTPClientResponseFetcher.class);
    bind(HttpClient.class).toProvider(HttpClientProvider.class);
    bind(PageURLParser.class).to(AhrefPageURLParser.class);

    // For parsing assets
    bind(AssetsParser.class).to(DefaultAssetsParser.class);
    bind(AssetsVerifier.class).to(DefaultAssetsVerifier.class);
    bind(AssetFetcher.class).to(HTTPClientAssetFetcher.class);

  }
530452
def dispatch(self, request, *args, **kwargs):
        
        self.args = args
        self.kwargs = kwargs
        request = self.initialize_request(request, *args, **kwargs)
        self.request = request
        self.headers = self.default_response_headers  # deprecate?

        try:
            self.initial(request, *args, **kwargs)

            # Get the appropriate handler method
            if request.method.lower() in self.http_method_names:
                handler = getattr(self, request.method.lower(),
                                  self.http_method_not_allowed)
            else:
                handler = self.http_method_not_allowed

            response = handler(request, *args, **kwargs)

        except Exception as exc:
            response = self.handle_exception(exc)

        self.response = self.finalize_response(request, response, *args, **kwargs)
        return self.response
880403
def set_default(self_,param_name,value):
        
        cls = self_.cls
        setattr(cls,param_name,value)
608631
def start_with(self, request):
        

        HTTPRequestHelper.patch_with_options(request, self.__options)
        self.queue.add_request(request)

        self.__crawler_start()
37201
def set_request_args(self, args):
        
        if self.scan_limit is not None:
            args['Limit'] = self.scan_limit
        elif self.item_limit is not None:
            args['Limit'] = max(self.item_limit, self.min_scan_limit)
        else:
            args.pop('Limit', None)
18997
def arg_default(*args, **kwargs):
    
    parser = argparse.ArgumentParser()
    parser.add_argument(*args, **kwargs)
    args = vars(parser.parse_args([]))
    _, default = args.popitem()
    return default
480342
def add_request_handler(self, request_handler):
        # type: (AbstractRequestHandler) -> None
        
        if request_handler is None:
            raise RuntimeConfigException(
                "Valid Request Handler instance to be provided")

        if not isinstance(request_handler, AbstractRequestHandler):
            raise RuntimeConfigException(
                "Input should be a RequestHandler instance")

        self.request_handler_chains.append(GenericRequestHandlerChain(
            request_handler=request_handler))
484606
def defaults(default=None):
    
    def _f(func):
        @functools.wraps(func)
        def __f(self, *args, **kwargs):
            try:
                return func(self, *args, **kwargs)
            except Exception:
                return default
        return __f
    return _f
237630
def process_kwargs(self, kwargs, prefix='default_', delete=True):
        
        processed = []
        for k in kwargs:
            if hasattr(self, prefix + k):
                processed += [k]
                setattr(self, prefix + k, kwargs[k])
        for k in processed:
            del(kwargs[k])
        return kwargs