Asynchronous request manager for session.  Returns list of responses that
    match the filter.

    `urls` - tuple of URLs to request  
    `re_filter` - a compiled regular expression
    [object](https://docs.python.org/3/library/re.html#re-objects)
497549
async def asynchronous(urls=None, re_filter=None):
    
    class _URLBase(str):
        """ Convenient access to hostname (ip) portion of the URL """
        @property
        def hostname(self):
            return urlsplit(self).hostname

    http_devices = {}
    qualified_devices = []
    connection = aiohttp.TCPConnector(limit=0)
    async with aiohttp.ClientSession(connector=connection, 
        conn_timeout=5, raise_for_status=True) as session:
        
        futures = [fetch_page(session, url) for url in urls]

        for future in asyncio.as_completed(futures):
            response = await future
            if 'found' in response[1]:
                http_devices[response[0]] = response[2]
                logger.debug('Processed %s', response[0])
                if re_filter.search(response[2]):
                    qualified_devices.append(_URLBase(response[0]))

    # print('The following responded to HTTP:')
    # for x in http_devices.keys():
    #     print(x)

    return qualified_devices
1038912
def _get_filter_ids(cls, request):
        
        id_query = request.url.query.get('id', None)

        if id_query is None:
            return None

        filter_ids = id_query.split(',')
        for filter_id in filter_ids:
            cls._validate_id(filter_id)

        return filter_ids
899074
def derived_get_requests(self):
        

        host = self.queue_item.response.url
        content = self.queue_item.response.text

        found_requests = []

        for expression in self.__expressions:
            matches = re.findall(expression["raw"], content)

            for match in matches:
                found_url = match[expression["group"]]
                absolute_url = URLHelper.make_absolute(host, found_url)
                found_requests.append(Request(absolute_url))

        return found_requests
320123
def get(self, request, bot_id, id, format=None):
        
        return super(UrlParameterList, self).get(request, bot_id, id, format)
967840
def get_urls(self, order="total_clicks desc", offset=None, count=None):
        
        req_data = [ None, order, fmt_paging(offset, count) ]
        return self.request("query:Message_Url", req_data)
272508
def _init_request_urls(self, api_urls):
        
        if not isinstance(api_urls, (str, list, tuple)):
            raise TypeError('api_urls needs to be string or iterable!')

        if isinstance(api_urls, str):
            api_urls = (api_urls,)

        api_urls = list(api_urls)

        for url in api_urls:
            if not url.startswith('http://') and \
               not url.startswith('https://'):
                raise ValueError(('URL "%s" contains an invalid or missing'
                                 ' scheme' % (url)))

        return list(api_urls)
343619
def get_urlpatterns(self):
        
        return [
            path(
                '',
                search_view_factory(view_class=self.search_view, form_class=self.search_form),
                name='search',
            ),
        ]
141135
def get_filter_ngrams_pattern(filter_ngrams):
        
        return re.compile('|'.join([re.escape(ngram) for ngram in
                                    filter_ngrams]))
578313
def _build_query_url(self, page = None, verbose = False):
        
        query = []
#        # build the filters
#        for afilter in self.filters.keys():
#            value = self.filters[afilter]
#            print"filter:%s value:%s" % (afilter,value)
#            value = urlencode(value)
#            query_str = u"%s=%s" % (afilter, value)
        if len(self.filters) > 0:
            query.append(urlencode(self.filters))
        if self.sort:
            query_str = u"%s=%s" % (u"sort", self.sort)
            query.append(query_str)
        if self.sort_by:
            query_str = u"%s=%s" % (u"sort_by", self.sort_by)
            query.append(query_str)
        if self.per_page:
            query_str = u"%s=%s" % (u"per_page", self.per_page)
            query.append(query_str)
        if page:
            query_str = u"%s=%s" % (u"page", page)
            query.append(query_str)
        query = u"?%s" % (u"&".join(query))
        url = u"%s%s" % (self.get_list_endpoint()['href'],query)
        url = u"%s%s%s" % (self.__api__.base_url, API_BASE_PATH, url)
        msg = "_build_query_url: url:%s" % url
        log.debug(msg)
        if verbose:
            print msg
        return url
652518
def _list(self, uri, obj_class=None, list_all=False):
        
        resp, resp_body = self._retry_get(uri)
        if obj_class is None:
            obj_class = self.resource_class

        data = resp_body[self.plural_response_key]
        ret = [obj_class(self, res, loaded=False)
                for res in data if res]
        self._reset_paging("domain", resp_body)
        if list_all:
            dom_paging = self._paging.get("domain", {})
            while dom_paging.get("next_uri"):
                next_uri = dom_paging.get("next_uri")
                ret.extend(self._list(uri=next_uri, obj_class=obj_class,
                        list_all=False))
        return ret
1022167
def _format(self):
        
        self.log.debug("Formatting URLs to request")
        items = list()
        for i in range(0, self.limit, 10):
            query = '"%s" %s' % (self.domain, self.modifier)
            url = self.host + "/search?q=" + query + "&first=" + str(i)
            items.append(url)
        self.log.debug("URLs were generated")
        return items
531560
def repeat_read_url_request(url, headers=None, data=None, retries=2, logger=None):
    '''
    
    '''
    if logger:
        logger.debug("Retrieving url content: {}".format(url))
    req = urllib2.Request(url, data, headers=headers or {})
    return repeat_call(lambda: urllib2.urlopen(req).read(), retries)
1253087
def _async_requests(urls):
    
    session = FuturesSession(max_workers=30)
    futures = [
        session.get(url)
        for url in urls
    ]
    return [ future.result() for future in futures ]
726197
def _get_sync(self, url):
        
        response = self.session.get(url)
        if response.status_code == requests.codes.ok:
            return response.json()
        else:
            raise HTTPError
1004381
def build_url_chunks(self):
        
        CHUNK_SIZE = 500

        id_filter = str(self.api_filters.pop(self.id_filter)).split(',')
        chuncks = list(self.chunks(id_filter, CHUNK_SIZE))
        filters = '&'.join("%s=%s" % (k, v) for (k, v) in self.api_filters.items())

        for chunk in chuncks:
            if filters:
                url = "{0}?{1}&{2}={3}".format(self.url, filters, self.id_filter, ','.join(chunk))
            else:
                url = "{0}?{1}={2}".format(self.url, self.id_filter, ','.join(chunk))
            self.split_urls.append(url)
811845
def do_get_block(req, proxy, conn, &block)
    @request_filter.each do |filter|
      filter.filter_request(req)
    end
    if str = @test_loopback_response.shift
      dump_dummy_request_response(req.body.dump, str) if @debug_dev
      conn.push(HTTP::Message.new_response(str))
      return
    end
    content = ''
    res = HTTP::Message.new_response(content)
    @debug_dev << "= Request\n\n" if @debug_dev
    sess = @session_manager.query(req, proxy)
    res.peer_cert = sess.ssl_peer_cert
    @debug_dev << "\n\n= Response\n\n" if @debug_dev
    do_get_header(req, res, sess)
    conn.push(res)
    sess.get_data() do |str|
      block.call(str) if block
      content << str
    end
    @session_manager.keep(sess) unless sess.closed?
    commands = @request_filter.collect { |filter|
      filter.filter_response(req, res)
    }
    if commands.find { |command| command == :retry }
      raise Client::RetryableResponse.new
    end
  end
910777
def _url_matches(self, url, other, match_querystring=False):
        '''
        
        '''
        if not match_querystring:
            other = other.split('?', 1)[0]

        if responses._is_string(url):
            if responses._has_unicode(url):
                url = responses._clean_unicode(url)
                if not isinstance(other, six.text_type):
                    other = other.encode('ascii').decode('utf8')
            return self._url_matches_strict(url, other)
        elif isinstance(url, responses.Pattern) and url.match(other):
            return True
        else:
            return False