retrieve and process any queued link
975976
function getLinkContents(err, queuedLink) {
  // no link found
  if (!queuedLink) {
    return;
  }
  // retrieve https directly
  var method = (queuedLink.uri.toString().indexOf('https') === 0) ? utils.retrieveHTTPS : utils.retrieve;

  method(queuedLink.uri, function(err, data) {
    contentLib.indexContentItem({ uri: ''+queuedLink.uri, content: data}, {isHTML: true, member: queuedLink.queued.member}, function(err, res, cItem) {
      if (err || !cItem) {
        GLOBAL.error('getLinkContents failed', err, res);
        return;
      }
      // if it recently gained content queue links
      if (cItem.state !== utils.states.content.queued && cItem.previousState === utils.states.content.queued) {
        GLOBAL.info('queueing links', cItem.uri, 'relevance', cItem.queued.relevance);
        queueLinks(cItem);
      }
      GLOBAL.debug('scraped', err, data ? data.length : 'no content');
    });
  });
}
69686
async def get_links_from_url(url):
    
    response = await httpclient.AsyncHTTPClient().fetch(url)
    print("fetched %s" % url)

    html = response.body.decode(errors="ignore")
    return [urljoin(url, remove_fragment(new_url)) for new_url in get_links(html)]
812178
function(err, res) {
    var queuedLink = null;
    if (res && res.hits && res.hits.hits.length) {
      var hit = res.hits.hits[0]._source;
      var queued = hit.queued;
      queuedLink = { uri: hit.uri, queued: queued, total: res.hits.total };
      queued.lastAttempt = new Date().toISOString();
      queued.attempts = queued.attempts + 1;
      hit.queued = queued;

      // update accesses
      contentLib.indexContentItem(hit, {state: 'queued', member: queued.member}, function(err, res) {
        // process the link
        if (!err) {
          callback(null, queuedLink);
        } else {
          utils.passingError(err);
        }
      });
    } else {
      callback(err, null);
    }
  }
859238
def get_queued():
    
    return Email.objects.filter(status=STATUS.queued) \
        .select_related('template') \
        .filter(Q(scheduled_time__lte=now()) | Q(scheduled_time=None)) \
        .order_by(*get_sending_order()).prefetch_related('attachments')[:get_batch_size()]
1233516
def get_queue_attributes(queue_url, attributes = ["All"])
      attributes = attributes.map(&:to_s)

      query = Hash[*attributes.each_with_index.map do |attribute, index|
        ["AttributeName.#{index+1}", attribute]
      end.flatten]

      action("GetQueueAttributes", queue_url, query)
    end
372807
def retrieve(cls, id, api_key=None, **params):
        """"""
        instance = cls(id, api_key, **params)
        d = instance.refresh()
        return d.addCallback(lambda _: instance)
314434
def do_GET(self):
    """"""
    if not self._client_allowed():
      return

    try:
      (_, _, path, query, _) = urlsplit(self.path)
      params = parse_qs(query)
      # Give each handler a chance to respond.
      for prefix, handler in self._GET_handlers:
        if self._maybe_handle(prefix, handler, path, params):
          return
      # If no path specified, default to showing the list of all runs.
      if path == '/':
        self._handle_runs('', {})
        return

      content = 'Invalid GET request {}'.format(self.path).encode('utf-8'),
      self._send_content(content, 'text/html', code=400)
    except (IOError, ValueError):
      pass
1237252
def get(self, loop=None):
        """"""
        self._clear_done_waiters()
        if not self._waiters:
            # If something is available directly, we can avoid going back to
            # the scheduler
            try:
                heap = self.get_nowait()
            except spead2.Empty:
                pass
            else:
                # Give the event loop a chance to run. This ensures that a
                # heap-processing loop cannot live-lock the event loop.
                yield
                raise Return(heap)

        if loop is None:
            loop = self._loop
        waiter = trollius.Future(loop=loop)
        self._waiters.append(waiter)
        self._start_listening()
        heap = (yield From(waiter)).pop()
        raise Return(heap)
369037
def get(block_id):
    """"""
    _url = get_root_url()
    try:
        block = DB.get_block_details([block_id]).__next__()
        response = block

        response['links'] = {
            'self': '{}'.format(request.url),
            'list': '{}/processing-blocks'.format(_url),
            'home': '{}'.format(_url)
        }
        return block
    except IndexError as error:
        response = dict(message='Unable to GET Processing Block',
                        id='{}'.format(block_id),
                        error=error.__str__())
        response['links'] = {
            'list': '{}/processing-blocks'.format(_url),
            'home': '{}'.format(_url)
        }
        return response, HTTPStatus.NOT_FOUND
76007
function handle_checkUrl(result)
			{
				maybeCallback(thisObj.handlers.link)(result, input.data.customData);
				
				// Auto-starts next queue item, if any
				// If not, fires "end"
				done();
			}
897558
def get_queue_func(request):
    """"""
    def cleanup(request):
        conn.close()

    def queue_func(**kwargs):
        return conn.channel().basic_publish(
            exchange='', body=json.dumps(kwargs), routing_key=queue,
            properties=pika.BasicProperties(delivery_mode=2))
    server = request.registry.settings['queue_server']
    queue = request.registry.settings['queue_verification']
    conn = pika.BlockingConnection(pika.ConnectionParameters(host=server))
    request.add_finished_callback(cleanup)
    return queue_func
859139
def get_paged_request(url, headers=None, **params):
    """"""
    results = []
    params.setdefault("per_page", 100)
    while True:
        if '?' in url:
            params = None
            print("fetching %s" % url, file=sys.stderr)
        else:
            print("fetching %s with %s" % (url, params), file=sys.stderr)
        response = requests.get(url, headers=headers, params=params)
        response.raise_for_status()
        results.extend(response.json())
        if 'next' in response.links:
            url = response.links['next']['url']
        else:
            break
    return results
968397
protected function _readUrlQueue() {
        $readQueue = @unserialize(
            Mage::app()->loadCache(self::CRAWLER_URLS_CACHE_ID) );
        if ( ! is_array($readQueue)) {
            // This is the first time the queue has been read since the last
            // cache flush (or the queue is corrupt)
            // Returning an empty array here would be the proper behavior,
            // but causes the queue to not be saved on the full cache flush event
            return $this->getAllUrls();
        } else {
            return $readQueue;
        }
    }
918545
def _get(self, *args, **kwargs):
        """"""
        prev_data = kwargs.pop('prev_data', [])
        resp = self._request('get', *args, **kwargs)
        resp_content = resp._content
        if not resp_content:
            # No content so its obviously not paginated
            return resp

        # if resp._content is a bytes object, decode it so we can loads it as json
        if isinstance(resp_content, bytes):
            resp_content = resp_content.decode('utf-8')

        # Load the json so we can use the data as python objects
        content = json.loads(resp_content)
        if 'pagination' not in content:
            # Result is not paginated
            return resp

        page_info = content['pagination']
        if not page_info['next_uri']:
            # next_uri is None when the cursor has been iterated to the last element
            content['data'].extend(prev_data)
            # If resp._content was is a bytes object, only set it as a bytes object
            if isinstance(resp_content, bytes):
                resp._content = json.dumps(content).decode('utf-8')
            else:
                resp._content = json.dumps(content)
            return resp

        prev_data.extend(content['data'])
        next_page_id = page_info['next_uri'].split('=')[-1]
        kwargs.update({
            'prev_data': prev_data,
            'params': {'starting_after': next_page_id}
        })
        return self._get(*args, **kwargs)
245343
function getPage(url, cb) {

    var page = generator.page$[u.urlPath(url)];
    var getFn = page && page.get && generator['get'+page.get];

    if (getFn) {
      getFn(page, function() { cb(null, page); });
    }
    else {
      process.nextTick(function() { cb(null, page); });
    }
  }
107536
def get(self, key, recursive=False, sorted=False, quorum=False,
            wait=False, wait_index=None, timeout=None):
        """"""
        url = self.make_key_url(key)
        params = self.build_args({
            'recursive': (bool, recursive or None),
            'sorted': (bool, sorted or None),
            'quorum': (bool, quorum or None),
            'wait': (bool, wait or None),
            'waitIndex': (int, wait_index),
        })
        if timeout is None:
            # Try again when :exc:`TimedOut` thrown.
            while True:
                try:
                    try:
                        res = self.session.get(url, params=params)
                    except:
                        self.erred()
                except (TimedOut, ChunkedEncodingError):
                    continue
                else:
                    break
        else:
            try:
                res = self.session.get(url, params=params, timeout=timeout)
            except ChunkedEncodingError:
                raise TimedOut
            except:
                self.erred()
        return self.wrap_response(res)
614554
protected void processLinkQueue()
    {
        long timeout = this.configuration.getCheckTimeout();
        List<Pattern> excludedReferencePatterns = this.configuration.getExcludedReferencePatterns();

        // Unqueue till we find an item that needs to be processed. We process an item if:
        // - it isn't present in the state map
        // - it is present but not enough time has elapsed since its last check time
        LinkQueueItem queueItem = null;
        boolean shouldBeChecked = false;

        while (!this.linkQueue.isEmpty() && !shouldBeChecked) {
            queueItem = this.linkQueue.poll();

            // Don't check excluded references
            shouldBeChecked = isExcluded(queueItem.getContentReference(), excludedReferencePatterns);
            if (!shouldBeChecked) {
                break;
            }

            Map<String, LinkState> contentReferences =
                this.linkStateManager.getLinkStates().get(queueItem.getLinkReference());
            if (contentReferences != null) {
                LinkState state = contentReferences.get(queueItem.getContentReference());
                if (state != null && (System.currentTimeMillis() - state.getLastCheckedTime() <= timeout)) {
                    shouldBeChecked = false;
                }
            }
        }

        if (shouldBeChecked && queueItem != null) {
            checkLink(queueItem);
        }
    }