Enqueue payloads to the SQS queue, retrying failed messages with
        exponential backoff.
198847
def send(self, payloads, logger, num_tries=5):
        
        from time import sleep

        backoff_interval = 1
        backoff_factor = 2

        for try_counter in xrange(0, num_tries):
            failed_messages = self.send_without_retry(payloads)

            # success!
            if not failed_messages:
                payloads = []
                break

            # output some information about the failures for debugging
            # purposes. we expect failures to be quite rare, so we can be
            # pretty verbose.
            if logger:
                for msg in failed_messages:
                    logger.warning("Failed to send message on try %d: Id=%r, "
                                   "SenderFault=%r, Code=%r, Message=%r" %
                                   (try_counter, msg['Id'],
                                    msg.get('SenderFault'), msg.get('Code'),
                                    msg.get('Message')))

            # wait a little while, in case the problem is that we're talking
            # too fast.
            sleep(backoff_interval)
            backoff_interval *= backoff_factor

            # filter out the failed payloads for retry
            retry_payloads = []
            for msg in failed_messages:
                i = int(msg['Id'])
                retry_payloads.append(payloads[i])
            payloads = retry_payloads

        if payloads:
            raise Exception('Messages failed to send to sqs after %d '
                            'retries: %s' % (num_tries, len(payloads)))
1295358
def retry(default=None):
    """"""

    def decorator(func):
        """Retry decorator"""

        @functools.wraps(func)
        def _wrapper(*args, **kw):
            for pos in range(1, MAX_RETRIES):
                try:
                    return func(*args, **kw)
                except (RuntimeError, requests.ConnectionError) as error:
                    LOGGER.warning("Failed: %s, %s", type(error), error)

                # Wait a bit before retrying
                for _ in range(pos):
                    _rest()

            LOGGER.warning("Request Aborted")
            return default

        return _wrapper

    return decorator
1274151
function(id, msg) {

        var q = queues[id], sc;
        if (q.timer) {
            // q.timer.cancel();
            clearTimeout(q.timer);
        }

        // execute failure callback
        if (q.onFailure) {
            sc = q.context || q;
            q.onFailure.call(sc, _returnData(q, msg));
        }

        _end(id, msg, 'failure');
    }
997573
def retry(self, **kwargs):
        
        path = '%s/%s/retry' % (self.manager.path, self.get_id())
        self.manager.gitlab.http_post(path)
1071933
def retry(self, retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504)):
        
        retries = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=status_forcelist,
        )
        # mount all https requests
        self.mount('https://', adapters.HTTPAdapter(max_retries=retries))
897090
def failed_dispatch(loader, queue, error):
    # type: (DataLoader, Iterable[Loader], Exception) -> None
    
    for l in queue:
        loader.clear(l.key)
        l.reject(error)
615943
def fail(exception)
      begin
        run_failure_hooks(exception)
      rescue Exception => e
        raise e
      ensure
        Failure.create \
          :payload   => payload,
          :exception => exception,
          :worker    => worker,
          :queue     => queue
      end
    end
234314
def fail_tmp(item, max_tries=None, ttl=None):
    ''''''
    try:
        max_tries = item.max_tries if max_tries is None else max_tries
        ttl = item.ttl if ttl is None else ttl

        # see if we need to fail perm
        check_ttl_max_tries(item.tries+1, item.enqueued_at, max_tries, ttl)
        # mv to same plus 1
        item.tries += 1
        new_name = construct(( fmt_time(item.enqueued_at, _c.FSQ_TIMEFMT,
                               _c.FSQ_CHARSET), item.entropy,
                               item.pid, item.hostname,
                               item.tries, ) + tuple(item.arguments))
        os.rename(fsq_path.item(item.queue, item.id, host=item.host),
				  fsq_path.item(item.queue, new_name, host=item.host))
        return new_name
    except (FSQMaxTriesError, FSQTTLExpiredError, FSQEnqueueError, ), e:
        fail_perm(item)
        e.strerror = u': '.join([
            e.strerror,
            u'for item {0}; failed permanently'.format(item.id),
        ])
        raise e
867629
def retry(self, msg, e, retry_num, waittime):
        ''' 
        '''
        self.msg.error_message(msg)
        self.msg.error_message(e)
        self.msg.error_message("Attempt number " + str(retry_num) + ". Retrying in " + str(waittime) + " seconds.")
        time.sleep(waittime)
949907
def requeue_delayed_jobs(self):
        
        failures = []
        for queue in self.queue_model.get_all_by_priority(self.queues):
            failures.extend(queue.requeue_delayed_jobs())

        self.last_requeue_delayed = datetime.utcnow()

        for failure in failures:
            self.log('Unable to requeue %s: %s' % failure)
1000657
def reversed_enqueue_task(task)
      RRRSpec.redis.lpush(RRRSpec.make_key(key, 'task_queue'), task.key)
    end
406269
def fail(self, exception):
        
        fail = failure.create(exception, self._queue, self._payload,
                              self._worker)
        fail.save(self.resq)
        return fail
623024
def defer_retry(message)
      sqs.change_message_visibility queue_url:          url_for_queue(message.queue),
                                    receipt_handle:     message.receipt_handle,
                                    visibility_timeout: BetterSqs.configuration.sqs_message_deferral_seconds
    end
900974
def __send_retry_requests(self, last_send_failure_time):
        """"""
        # make sure multiple failures having set multiple times do not run concurrently
        with self.__send_retry_requests_lock:
            with self.__requests:
                # produce list instead of generator as requests mapping can change during subsequent loop
                retry_reqs = [req for req in self.__requests.values()
                              if req._sent_without_response(last_send_failure_time)]

            retry_req_count = 0
            # don't continue if another network failure has occured (which will trigger this function again)
            while retry_reqs and self.__amqplink.last_send_exc_time <= last_send_failure_time:
                req = retry_reqs.pop()
                # lock individuallly so incoming request handling does not 'pause' for too long
                with self.__requests:
                    # might have received a response (or finished since)
                    if not (req.id_ in self.__requests and req._sent_without_response(last_send_failure_time)):
                        logger.debug('Not resending request %s (finished or has received response)', req.id_)
                        continue
                logger.debug('Resending request %s', req.id_)
                if not self.__retry_enqueue(PreparedMessage(req._inner_msg_out, req.id_)):
                    # client shutdown
                    break
                retry_req_count += 1

        if retry_req_count:
            logger.debug('Resending of %d request(s) complete (before %s)', retry_req_count, last_send_failure_time)
748026
def stash(self, payload):
        """"""
        succeeded = []
        failed = []
        for key in payload['keys']:
            if self.queue.get(key) is not None:
                if self.queue[key]['status'] == 'queued':
                    self.queue[key]['status'] = 'stashed'
                    succeeded.append(str(key))
                else:
                    failed.append(str(key))
            else:
                failed.append(str(key))

        message = ''
        if len(succeeded) > 0:
            message += 'Stashed entries: {}.'.format(', '.join(succeeded))
            status = 'success'
        if len(failed) > 0:
            message += '\nNo queued entry for keys: {}'.format(', '.join(failed))
            status = 'error'

        answer = {'message': message.strip(), 'status': status}

        return answer
658138
def reset_queues(queues):
    """"""
    for queue in queues:
        with queue.mutex:
            queue._put = queue._pebble_old_method
        delattr(queue, '_pebble_old_method')
        delattr(queue, '_pebble_lock')
333568
def with_retries(*rescued, &block)
      options      = rescued.last.is_a?(Hash) ? rescued.pop : {}
      exception    = nil
      retries      = 0

      rescued = Defaults::RETRIED_EXCEPTIONS if rescued.empty?

      max_attempts = options[:attempts] || Defaults::RETRY_ATTEMPTS
      backoff_base = options[:base]     || Defaults::RETRY_BASE
      backoff_max  = options[:max_wait] || Defaults::RETRY_MAX_WAIT

      begin
        return yield retries, exception
      rescue *rescued => e
        exception = e

        retries += 1
        raise if retries > max_attempts

        # Calculate the exponential backoff combined with an element of
        # randomness.
        backoff = [backoff_base * (2 ** (retries - 1)), backoff_max].min
        backoff = backoff * (0.5 * (1 + Kernel.rand))

        # Ensure we are sleeping at least the minimum interval.
        backoff = [backoff_base, backoff].max

        # Exponential backoff.
        Kernel.sleep(backoff)

        # Now retry
        retry
      end
    end