Setup redis connection and idle signal.

        This should be called after the spider has set its crawler object.
487278
def setup_redis(self, crawler=None):
        
        if self.server is not None:
            return

        if crawler is None:
            # We allow optional crawler argument to keep backwards
            # compatibility.
            # XXX: Raise a deprecation warning.
            crawler = getattr(self, 'crawler', None)

        if crawler is None:
            raise ValueError("crawler is required")

        settings = crawler.settings

        if self.redis_key is None:
            self.redis_key = settings.get(
                'REDIS_START_URLS_KEY', defaults.START_URLS_KEY,
            )

        self.redis_key = self.redis_key % {'name': self.name}

        if not self.redis_key.strip():
            raise ValueError("redis_key must not be empty")

        if self.redis_batch_size is None:
            # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
            self.redis_batch_size = settings.getint(
                'REDIS_START_URLS_BATCH_SIZE',
                settings.getint('CONCURRENT_REQUESTS'),
            )

        try:
            self.redis_batch_size = int(self.redis_batch_size)
        except (TypeError, ValueError):
            raise ValueError("redis_batch_size must be an integer")

        if self.redis_encoding is None:
            self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)

        self.logger.info("Reading start URLs from redis key '%(redis_key)s' "
                         "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
                         self.__dict__)

        self.server = connection.from_settings(crawler.settings)
        # The idle signal is called when the spider has no requests left,
        # that's when we will schedule new requests from redis queue
        crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)
1157568
def run_ajax_spider(self, target_url):
        """"""
        self.logger.debug('AJAX Spidering target {0}...'.format(target_url))

        self.zap.ajaxSpider.scan(target_url)

        while self.zap.ajaxSpider.status == 'running':
            self.logger.debug('AJAX Spider: {0}'.format(self.zap.ajaxSpider.status))
            time.sleep(self._status_check_sleep)

        self.logger.debug('AJAX Spider completed')
718858
def init_db_conn(
        connection_name, HOST=None, PORT=None, DB=None, PASSWORD=None):
    
    rpool = redis.ConnectionPool(
        host=HOST, port=PORT, db=DB, password=PASSWORD)
    r = redis.Redis(connection_pool=rpool)
    redis_pool.connections[connection_name] = RedisClient(r)
902519
public function onReady()
    {
        $this->redis = \PHPDaemon\Clients\Redis\Pool::getInstance();

        $this->redis->multi(function ($multi) {
            // "OK"
            D('start multi: ' . $multi->result);

            $multi->set('test1', 'value1', function ($redis) use ($multi) {
                // "QUEUED"
                D('in multi 1: ' . $redis->result);

                $this->redis->set('test1', 'value1-new', function ($redis) {
                    // "OK", not "QUEUED"
                    D('out multi 1: ' . $redis->result);
                });

                setTimeout(function ($timer) use ($multi) {
                    // "QUEUED"
                    $multi->set('test2', 'value2', function ($redis) use ($multi) {
                        D('in multi 2: ' . $redis->result);

                        $multi->exec(function ($redis) {
                            D('exec');
                            D($redis->result);
                        });
                    });
                    $timer->free();
                }, 2e5);
            });
        });
    }
1054012
def wait_until_idle(self):
        

        done = threading.Event()

        def _callback():
            done.set()

        self.defer_until_idle(_callback)
        done.wait()
172309
function synchronize() {
    if (syncPointer != -1) {
        log.logMessage("verbose", "redis synchronization!");
        var syncMessage = {};
        syncMessage.data = messages.splice(syncPointer, messages.length - syncPointer);
        setTimeout(function () {
            pub.publish("message", JSON.stringify(syncMessage));
        }, 200);
        syncPointer = -1;
    }
}
776822
def create_connection():
    """"""
    # REDIS_URL is defined in .env and loaded into the environment by Honcho
    redis_url = os.getenv('REDIS_URL')
    # If it's not defined, use the Redis default
    if not redis_url:
        redis_url = 'redis://localhost:6379'
    urlparse.uses_netloc.append('redis')
    url = urlparse.urlparse(redis_url)
    return redis.StrictRedis(
        host=url.hostname,
        port=url.port,
        db=0,
        password=url.password
    )
1063730
def _spawn_redis_connection_thread(self):
        """"""
        self.logger.debug("Spawn redis connection thread")
        self.redis_connected = False
        self._redis_thread = Thread(target=self._setup_redis)
        self._redis_thread.setDaemon(True)
        self._redis_thread.start()
602946
protected function connect()
    {
        if (!is_object(self::$instance)) {
            $func = $this->options['persistent'] ? 'pconnect' : 'connect';

            self::$instance = new \Redis();
            self::$instance->$func($this->options['host'], $this->options['port'], $this->options['timeout']);

            if ('' != $this->options['password']) {
                self::$instance->auth($this->options['password']);
            }
            if (0 != $this->options['select']) {
                self::$instance->select($this->options['select']);
            }
        }
    }
1043553
private void complete() {
		if (stopped) {
			return;
		}

		log.info("Spidering process is complete. Shutting down...");
		this.stopped = true;
		if (httpSender != null) {
			this.getHttpSender().shutdown();
			httpSender = null;
		}

		// Notify the controller to clean up memory
		controller.reset();

		// Issue the shutdown command on a separate thread, as the current thread is most likely one
		// from the pool
		new Thread(new Runnable() {
			@Override
			public void run() {
				if (threadPool != null) {
					threadPool.shutdown();
				}
				// Notify the listeners -- in the meanwhile
				notifyListenersSpiderComplete(true);
				controller.reset();
				threadPool = null;
			}
		}, "ZAP-SpiderShutdownThread-" + id).start();
	}
247358
def set(self, key, value):
        """"""
        value = pickle.dumps(value, protocol=constants.PICKLE_PROTOCOL)
        redis_conn.set(key, value)
1172315
async def setup(self):
        """"""
        while True:
            fut = self.loop.create_connection(
                lambda: SW16Protocol(
                    self,
                    disconnect_callback=self.handle_disconnect_callback,
                    loop=self.loop, logger=self.logger),
                host=self.host,
                port=self.port)
            try:
                self.transport, self.protocol = \
                    await asyncio.wait_for(fut, timeout=self.timeout)
            except asyncio.TimeoutError:
                self.logger.warning("Could not connect due to timeout error.")
            except OSError as exc:
                self.logger.warning("Could not connect due to error: %s",
                                    str(exc))
            else:
                self.is_connected = True
                if self.reconnect_callback:
                    self.reconnect_callback()
                break
            await asyncio.sleep(self.reconnect_interval)
536998
public function onReady()
    {
        $this->redis = \PHPDaemon\Clients\Redis\Pool::getInstance();

        $this->redis->hmset('myset', 'field1', 'value1', 'field2', 'value2', 'field3', 'value3', 'field4', 'value4',
            'field5', 'value5', function ($redis) {
                $this->redis->hgetall('myset', function ($redis) {
                    D('TEST 1: HGETALL');
                    foreach ($redis as $key => $value) {
                        D($key . ' - ' . $value);
                    }
                    D($redis->assoc);
                });

                $this->redis->hmget('myset', 'field2', 'field4', function ($redis) {
                    D('TEST 2: HMGET');
                    foreach ($redis as $key => $value) {
                        D($key . ' - ' . $value);
                    }
                    D($redis->assoc);
                });
            });

        $this->redis->zadd('myzset', 100, 'one', 150, 'two', 325, 'three', function ($redis) {
            $this->redis->zrange('myzset', 0, -1, function ($redis) {
                D('TEST 3: ZRANGE');
                foreach ($redis as $key => $value) {
                    D($key . ' - ' . $value);
                }
                D($redis->assoc);
            });

            $this->redis->zrange('myzset', 0, -1, 'WITHSCORES', function ($redis) {
                D('TEST 4: ZRANGE WITHSCORES');
                foreach ($redis as $key => $value) {
                    D($key . ' - ' . $value);
                }
                D($redis->assoc);
            });
        });

        $this->redis->subscribe('mysub', function ($redis) {
            D('TEST 5: SUB & PUB');
            foreach ($redis as $key => $value) {
                D($key . ' - ' . $value);
            }
            D($redis->channel, $redis->msg, $redis->assoc);
        });

        $this->redis->publish('mysub', 'Test message!');
    }
569706
def setup_signals
      Kernel.trap('INT')    { Thread.new { @shutdown.call(Signal.list['INT']) }.join }
      Kernel.trap('TERM')   { Thread.new { @shutdown.call(Signal.list['TERM']) }.join }
      unless RbConfig::CONFIG['host_os'] =~ /mswin|windows|cygwin/i
        Kernel.trap('HUP')  { Thread.new { @reload.call }.join }
      end
    end
1133125
function () {
        clearTimeout(this._conn._idleTimeout);
        this._throttledRequestHandler();
        this._conn._idleTimeout = setTimeout(this._conn._onIdle.bind(this._conn), 100);
    }
1131008
def redis_client(address=None, connection_pool=None, timeout=None,
                 parser=None, **kwargs):
    '''
    '''
    if not connection_pool:
        if timeout == 0:
            if not async:
                raise ImportError('Asynchronous connection requires async '
                                  'bindings installed.')
            return async.pool.redis(address, **kwargs)
        else:
            kwargs['socket_timeout'] = timeout
            return Redis(address[0], address[1], **kwargs)
    else:
        return Redis(connection_pool=connection_pool)
315497
def create_connection():
    """"""

    global _cached_connection
    settings = oz.settings

    if settings["redis_cache_connections"] and _cached_connection != None:
        return _cached_connection
    else:
        conn = redis.StrictRedis(
            host=settings["redis_host"],
            port=settings["redis_port"],
            db=settings["redis_db"],
            password=settings["redis_password"],
            decode_responses=settings["redis_decode_responses"],
            ssl=settings["redis_use_ssl"],
            ssl_keyfile=settings["redis_ssl_keyfile"],
            ssl_certfile=settings["redis_ssl_certfile"],
            ssl_cert_reqs=settings["redis_ssl_cert_reqs"],
            ssl_ca_certs=settings["redis_ssl_ca_certs"]
        )

        if settings["redis_cache_connections"]:
            _cached_connection = conn

        return conn