Simple rate limit function using redis
194674
def rate_limit(self):
        
        rate_limited_msg = False

        while True:
            is_rate_limited = self.limit.is_rate_limited(uuid)
            if is_rate_limited:
                time.sleep(0.3)  # save hammering redis
                if not rate_limited_msg:
                    self.logger.info('Rate limit active..please wait...')
                    rate_limited_msg = True

            if not is_rate_limited:
                self.logger.info('Rate limit clear.')
                self.limit.attempt(uuid)
                return True
192115
def limit(value = nil, &block)
      chain { criteria.update_request_options size: block || Integer(value) }
    end
818018
def rate_limiting_resettime(self):
        
        if self.__requester.rate_limiting_resettime == 0:
            self.get_rate_limit()
        return self.__requester.rate_limiting_resettime
809665
def rate_limit(client, headers, atexit=False):
    """"""
    if not client or not headers:
        return False

    if not getattr(client.config, "rate_limit", False):
        return False

    rate_info = RateLimitsInfo.from_headers(headers)
    if not rate_info or not rate_info.interval:
        return False

    if rate_info.interval:
        cb = getattr(client.config, "rate_limit_callback", None)
        if cb and callable(cb):
            cb(rate_info, atexit=atexit)
        time.sleep(rate_info.interval)
    return True
798919
function crawl(options, callback) {

          // Case of retries
          if (options.currentRetries > 0) {
            log.error({"url" : options.url, "step" : "queue-resquester.crawl", "message" : "Retry the request with a delay : " + options.retryTimeout});
            setTimeout(function() {
              execHttp(options, callback);
            }, options.retryTimeout);
            return;
          }

          // Case of a ratelimit forcing
          if (options.rateLimits !== 0) {

              log.error({"url" : options.url, "step" : "queue-resquester.crawl", "message" : "Request with option on ratelimit = " + options.rateLimits});
              setTimeout(function() {
                execHttp(options, callback);
              }, options.rateLimits);

          }
          // The usual case without delay
          else {
            execHttp(options, callback);
          }


  }
962732
function startAllConnections() {
  try {
    myUtils.validateConfig();
  }
  catch(e) {
    console.error(e.message);
    process.exit(2);
  }

  // redefine keys method before connections are started
  if (config.get('redis.useScan')) {
    console.log('Using scan instead of keys');
    Object.defineProperty(Redis.prototype, 'keys', {
      value: function(pattern, cb) {
        let keys = [];
        let that = this;
        let scanCB = function(err, res) {
          if (err) {
            if (typeof cb === 'function') cb(err);
            else {
              console.log('ERROR in redefined "keys" function to use "scan" instead without callback: ' + JSON.stringify(err));
            }
          }
          else {
            let count = res[0], curKeys = res[1];
            console.log("scanning: " + count + ": " + curKeys.length);
            keys = keys.concat(curKeys);
            if (Number(count) === 0) {
              if (typeof cb === 'function') cb(null, keys);
              else {
                console.log('ERROR in redefined "keys" function to use "scan" instead - no callback given!');
              }
            }
            else {
              that.scan(count, 'MATCH', pattern, 'COUNT', config.get('redis.scanCount'), scanCB);
            }
          }
        };
        return this.scan(0, 'MATCH', pattern, 'COUNT', config.get('redis.scanCount'), scanCB);
      }
    });
  }

  // first default connections from config object
  // second connection from cli params (redis-host, redis-port, ...)
  startDefaultConnections(config.connections, function (err) {
    if (err) {
      console.log(err);
      process.exit();
    }
    let db = parseInt(args['redis-db']);
    if (isNaN(db)) {
        db = 0;
    }

    let client;
    if (args['sentinel-host'] || args['sentinels'] || args['redis-host'] || args['redis-port'] || args['redis-socket'] || args['redis-password']) {
      let newDefault = {
        "label": config.get('redis.defaultLabel'),
        "dbIndex": db,
        "password": args['redis-password'] || '',
        "connectionName": config.get('redis.connectionName')
      };

      if (args['redis-socket']) {
        newDefault.path = args['redis-socket'];
      }
      else {
        newDefault.host = args['redis-host'] || "localhost";
        newDefault.port = args['redis-port'] || "6379";
        newDefault.sentinel_name = args['sentinel-name'];
        if (args['sentinels']) {
          newDefault.sentinels = myUtils.parseRedisSentinel('--sentinels', args['sentinels']);
        }
        else if (args['sentinel-host']) {
          newDefault.sentinels = myUtils.parseRedisSentinel( '--sentinel-host or --sentinel-port',
            args['sentinel-host'] + ':' + args['sentinel-port']);
        }
      }

      if (args['redis-tls']) {
        newDefault.tls = {};
      }

      if (!myUtils.containsConnection(config.connections, newDefault)) {
        client = myUtils.createRedisClient(newDefault);
        redisConnections.push(client);
        config.connections.push(newDefault);
        if (!config.get('noSave')) {
          myUtils.saveConnections(config,function (err) {
            if (err) {
              console.log("Problem saving connection config.");
              console.error(err);
            }
          });
        }
        setUpConnection(client, db);
      }
    } else if (config.connections.length === 0) {
      // fallback to localhost if nothing else configured
      client = myUtils.createRedisClient({label: config.get('redis.defaultLabel')});
      redisConnections.push(client);
      setUpConnection(client, db);
    }

    return startWebApp();
  });

  // wait a bit before starting browser to let http server start
  if (args['open']) {
    setTimeout(function() {
      let address = '127.0.0.1';
      if (config.get('server.address') !== '0.0.0.0' && config.get('server.address') !== '::') {
        address = config.get('server.address');
      }
      require('opener')('http://' + address + ':' + config.get('server.port'));
    }, 1000);
  }
}
738207
def rpush_limit(self, queue, value, limit=100000):
        ''' 
        '''

        lua = '''
        local queue = KEYS[1]
        local max_size = tonumber(KEYS[2])
        local table_len = 1
        local redis_queue_len = tonumber(redis.call('LLEN', queue))
        local total_size = redis_queue_len + table_len
        local from = 0

        if total_size >= max_size then
            -- Delete the same amount of data we are inserting. Even better, limit the queue to the specified size
            redis.call('PUBLISH', 'DEBUG', 'trim')
            if redis_queue_len - max_size + table_len > 0 then
                from = redis_queue_len - max_size + table_len
            else
                from = 0
            end
            redis.call('LTRIM', queue, from, redis_queue_len)
        end
        redis.call('RPUSH', queue, ARGV[1])
        return 1

        '''

        try:
            self.rpush_limit_script([queue, limit], [value])
        except AttributeError:
            if self.logger:
                self.logger.info('Script not registered... registering')
            # If the script is not registered, register it
            self.rpush_limit_script = self.register_script(lua)
            self.rpush_limit_script([queue, limit], [value])
654949
def rate_limit(self, rate_limit):
        
        self._rate_limit = bool(rate_limit)
        self._rate_limit_last_call = None
        self.clear_memoized()
1264918
function doMulti(connObj, cmds, callback) {
    if (!this.isConnected()) {
      return callback(thorin.error('REDIS.NOT_CONNECTED', 'Redis connection is not ready.', 500));
    }
    if (cmds.length === 0) return callback();
    // check commands first.
    for (let i = 0; i < cmds.length; i++) {
      let cmd = cmds[i][0];
      if (typeof connObj[cmd] !== 'function') {
        return callback(thorin.error('REDIS.COMMAND_NOT_FOUND', 'Invalid redis command:' + cmd, 500));
      }
    }
    var mObj = connObj.multi();
    cmds.forEach((item) => {
      let cmd = item.splice(0, 1)[0];
      mObj[cmd].apply(mObj, item);
      item.splice(0, 0, cmd);
    });
    this[connections].default.emit('multi', cmds);
    mObj.exec((err, results) => {
      if (err) {
        return callback(thorin.error('REDIS.MULTI', 'Redis transaction encountered an error.', err, 500));
      }
      let finalResults = [];
      for (let i = 0, len = results.length; i < len; i++) {
        let t = results[i];
        if (t[0] === null) {
          finalResults.push(t[1]);
        }
      }
      callback(null, finalResults);
    });
  }
881540
def count_between(self, min_score=None, max_score=None):
        
        min_score = float('-inf') if min_score is None else float(min_score)
        max_score = float('inf') if max_score is None else float(max_score)

        return self.redis.zcount(self.key, min_score, max_score)
939068
def sleep_and_retry(func):
    '''
    
    '''
    @wraps(func)
    def wrapper(*args, **kargs):
        '''
        Call the rate limited function. If the function raises a rate limit
        exception sleep for the remaing time period and retry the function.

        :param args: non-keyword variable length argument list to the decorated function.
        :param kargs: keyworded variable length argument list to the decorated function.
        '''
        while True:
            try:
                return func(*args, **kargs)
            except RateLimitException as exception:
                time.sleep(exception.period_remaining)
    return wrapper
583964
def get_global_rate_limit(self):
        
        r = urllib.request.urlopen('https://archive.org/metadata/iamine-rate-limiter')
        j = json.loads(r.read().decode('utf-8'))
        return int(j.get('metadata', {}).get('rate_per_second', 300))
728391
def _translate_limit(self, len_, start, num):
        
        if start > len_ or num <= 0:
            return 0, 0
        return min(start, len_), num
1213070
def maybe_rate_limit(client, headers, atexit=False):
    """"""
    # pylint: disable=fixme
    # pylint: disable=global-statement
    # FIXME: Yes, I know this is not great. We'll fix it later. :-)
    global LAST_CLIENT, LAST_HEADERS

    if LAST_CLIENT and LAST_HEADERS:
        # Wait based on previous client/headers
        rate_limit(LAST_CLIENT, LAST_HEADERS, atexit=atexit)

    LAST_CLIENT = copy.copy(client)
    LAST_HEADERS = copy.copy(headers)
362614
def rate_limited(max_per_hour: int, *args: Any) -> Callable[..., Any]:
    """"""
    return util.rate_limited(max_per_hour, *args)
539803
function wrapRemoteCache(remoteCache) {
    return {
        get: function (key) {
            var deferred = Q.defer();

            remoteCache.get(key, function (err, value) {
                if (err) {
                    console.log('redis remoteCache.get err ' + err);
                    deferred.resolve();
                }

                if (_.isString(value) && value.charAt(0) === '{') {
                    value = JSON.parse(value);
                }

                deferred.resolve(value);
            });

            return deferred.promise;
        },
        set: function (key, value) {
            value = _.isObject(value) ? JSON.stringify(value) : value;
            remoteCache.set(key, value);
            remoteCache.expire(key, 3600);
        },
        del: function (key) {
            remoteCache.del(key);
        },
        quit: function () {
            remoteCache.quit();
        },
        flush: function () {
            remoteCache.flushdb();
        }
    };
}
1267052
def is_rate_limited(response):
        
        if (response.status_code == codes.too_many_requests and 'Retry-After' in response.headers and
                int(response.headers['Retry-After']) >= 0):
            return True

        return False