encapsulates urllib retrieval for fetching JSON results from
        Google's Custom Search API.  Returns a deserialized result set.
969790
def web_search(self, query, start=0, limit=100, max_tries=3):
        '''
        
        '''
        tries = 0
        if isinstance(query, unicode):
            query = query.encode('utf8')
        url = self.url % dict(key=self.api_key,
                              query=urllib.quote(query.strip()),
                              num=min(10, limit - start),
                              start=start)
        logger.info("fetching: %s" % url)
        while 1:
            try:
                request = urllib2.Request(url)
                request.add_header('Accept-encoding', 'gzip')
                # We do we set this?  Remnant from pre-API version?
                request.add_header('User-Agent', USER_AGENT)
                opener = urllib2.build_opener()

                fh = opener.open(request, timeout=60)
                data = fh.read()
                if fh.headers.get('Content-Encoding') == 'gzip':
                    compressedstream = StringIO.StringIO(data)
                    fh = gzip.GzipFile(fileobj=compressedstream)
                    data = fh.read()
                return json.loads(data)
            except Exception, exc:
                logger.info(traceback.format_exc(exc))
                if tries >= max_tries:
                    sys.exit("failed %d times to fetch %s" % (max_tries, url))
                else:
                    logger.info("failed to fetch\n\t%s\nwill try "
                                "%d more times" % (url, max_tries - tries))
                    tries += 1
                    time.sleep(2 ** tries)
918359
def get_data(search_string, search_by='ip'):
    
    r_details = requests.get('https://talosintelligence.com/sb_api/query_lookup',
            headers={
                'Referer':'https://talosintelligence.com/reputation_center/lookup?search=%s'%search_string,
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0'
                },
            params = {
                'query':'/api/v2/details/ip/',
                'query_entry':search_string
                }).json()

    r_wscore = requests.get('https://talosintelligence.com/sb_api/remote_lookup',
            headers={
                'Referer':'https://talosintelligence.com/reputation_center/lookup?search=%s'%search_string,
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0'
                },
            params = {'hostname':'SDS', 'query_string':'/score/wbrs/json?url=%s' % search_string}).json()

    r_talos_blacklist = requests.get('https://www.talosintelligence.com/sb_api/blacklist_lookup',
            headers={
                'Referer':'https://talosintelligence.com/reputation_center/lookup?search=%s'%search_string,
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:64.0) Gecko/20100101 Firefox/64.0'
                },
            params = {'query_type':'ipaddr', 'query_entry':search_string}).json()

    # would be nice to plot this values
    #r_volume = requests.get('https://talosintelligence.com/sb_api/query_lookup',
    #        params = {
    #            'query':'/api/v2/volume/ip/',
    #            'query_entry':search_string
    #            }).json()

    # No used for now
    #r_related_ips = requests.get('https://talosintelligence.com/sb_api/query_lookup',
    #        params = {
    #            'query':'/api/v2/related_ips/ip/',
    #            'query_entry':search_string
    #            }).json()

    
    talos_blacklisted = {'status':False}
    if 'classifications' in r_talos_blacklist['entry']:
        talos_blacklisted['status'] = True
        talos_blacklisted['classifications'] = ", ".join(r_talos_blacklist['entry']['classifications'])
        talos_blacklisted['first_seen'] = r_talos_blacklist['entry']['first_seen'] + "UTC"
        talos_blacklisted['expiration'] = r_talos_blacklist['entry']['expiration'] + "UTC"

    data = {
        'address':search_string,
        'hostname':r_details['hostname'] if 'hostname' in r_details else "nodata",
        'volume_change':r_details['daychange'] if 'daychange' in r_details else "nodata",
        'lastday_volume':r_details['daily_mag'] if 'daily_mag' in r_details else "nodata",
        'month_volume':r_details['monthly_mag'] if 'monthly_mag' in r_details else "nodata",
        'email_reputation':r_details['email_score_name'] if 'email_score_name' in r_details else "nodata",
        'web_reputation':r_details['web_score_name'] if 'web_score_name' in r_details else "nodata",
        'weighted_reputation_score':r_wscore['response'],
        'talos_blacklisted':"Yes" if talos_blacklisted['status'] else "No"
        #'weighted_reputation_score':r_wscore[0]['response']['wbrs']['score'],
        #'volumes':zip(*r_volume['data'])
    }

    return data
267613
def __fetch_data(self, url):
        
        url += '&api_key=' + self.api_key
        try:
            response = urlopen(url)
            root = ET.fromstring(response.read())
        except HTTPError as exc:
            root = ET.fromstring(exc.read())
            raise ValueError(root.get('message'))
        return root
1244103
def get(self, url, params=dict()):
        
        try:
            res = requests.get(url, headers=self.headers, params=params)
            return json.loads(res.text)
        except Exception as e:
            print(e)
            return "error"
599996
def get_results(self, client):
        
        data = self._request('GET', '/results/{}'.format(client))
        return data.json()
268299
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in search.finditer(data[0]):
                searchUrl = match.group(1)
                if not searchUrl:
                    raise ValueError("Pattern %s matched empty URL at %s." % (search.pattern, url))
                out.debug(u'matched URL %r with pattern %s' % (searchUrl, search.pattern))
                searchUrls.append(normaliseURL(urljoin(data[1], searchUrl)))
            if searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            patterns = [x.pattern for x in searches]
            raise ValueError("Patterns %s not found at URL %s." % (patterns, url))
        return searchUrls
670628
def _fetch(self, searchtype, fields, **kwargs):
        ''''''
        fields['vintage'] = self.vintage
        fields['benchmark'] = self.benchmark
        fields['format'] = 'json'

        if 'layers' in kwargs:
            fields['layers'] = kwargs['layers']

        returntype = kwargs.get('returntype', 'geographies')
        url = self._geturl(searchtype, returntype)

        try:
            with requests.get(url, params=fields, timeout=kwargs.get('timeout')) as r:
                content = r.json()
                if "addressMatches" in content.get('result', {}):
                    return AddressResult(content)

                if "geographies" in content.get('result', {}):
                    return GeographyResult(content)

                raise ValueError()

        except (ValueError, KeyError):
            raise ValueError("Unable to parse response from Census")

        except RequestException as e:
            raise e
908420
def repeat_read_json_url_request(url, headers=None, data=None, retries=2, logger=None):
    '''
    
    '''
    if logger:
        logger.debug("Retrieving url json content: {}".format(url))
    req = urllib2.Request(url, data=data, headers=headers or {})
    return repeat_call(lambda: json.loads(urllib2.urlopen(req).read()), retries)
135685
def fetch_json(self, method, url, data=None, expected_status_code=None):
        
        return self.fetch(method, url, data, expected_status_code).json()
62733
def get_json(request, token):
    """"""
    result = []
    searchtext = request.GET['q']
    if len(searchtext) >= 3:
        pickled = _simple_autocomplete_queryset_cache.get(token, None)
        if pickled is not None:
            app_label, model_name, query = pickle.loads(pickled)
            model = apps.get_model(app_label, model_name)
            queryset = QuerySet(model=model, query=query)
            fieldname = get_search_fieldname(model)
            di = {'%s__istartswith' % fieldname: searchtext}
            app_label_model = '%s.%s' % (app_label, model_name)
            max_items = get_setting(app_label_model, 'max_items', 10)
            items = queryset.filter(**di).order_by(fieldname)[:max_items]

            # Check for duplicate strings
            counts = {}
            for item in items:
                if hasattr(item, "__unicode__"):
                    key = item.__unicode__()
                else:
                    key = str(item)
                #key = unicode(item)
                counts.setdefault(key, 0)
                counts[key] += 1

            # Assemble result set
            for item in items:
                #key = value = unicode(item)
                if hasattr(item, "__unicode__"):
                    key = value = item.__unicode__()
                else:
                    key = value = str(item)
                value = getattr(item, fieldname)
                if counts[key] > 1:
                    func = get_setting(
                        app_label_model,
                        'duplicate_format_function',
                        lambda obj, model, content_type: content_type.name
                    )
                    content_type = ContentType.objects.get_for_model(model)
                    value = '%s (%s)' % (value, func(item, model, content_type))
                result.append((item.id, value))

        else:
            result = 'CACHE_MISS'

    return HttpResponse(json.dumps(result))
704468
def _get_json(self, model, space=None, rel_path=None, extra_params=None, get_all=None):
        
        # Only API.spaces and API.event should not provide
        # the `space argument
        if space is None and model not in (Space, Event):
            raise Exception(
                'In general, `API._get_json` should always '
                'be called with a `space` argument.'
            )

        if not extra_params:
            extra_params = {}

        # Handle pagination for requests carrying large amounts of data
        extra_params['page'] = extra_params.get('page', 1)

        # Generate the url to hit
        url = '{0}/{1}/{2}.json?{3}'.format(
            settings.API_ROOT_PATH,
            settings.API_VERSION,
            rel_path or model.rel_path,
            urllib.urlencode(extra_params),
        )

        # If the cache is being used and the url has been hit already
        if self.cache_responses and url in self.cache:
            response = self.cache[url]
        else:
            # Fetch the data
            headers = {
                'X-Api-Key': self.key,
                'X-Api-Secret': self.secret,
            }
            response = self.session.get(url=url, headers=headers)
            # If the cache is being used, update it
            if self.cache_responses:
                self.cache[url] = response

        if response.status_code == 200:  # OK
            results = []
            json_response = response.json()
            for obj in json_response:
                instance = model(data=obj)
                instance.api = self
                if space:
                    instance.space = space
                results.append(instance)
            # If it looks like there are more pages to fetch,
            # try and fetch the next one
            per_page = extra_params.get('per_page', None)
            if (
                get_all
                and per_page
                and len(json_response)
                and per_page == len(json_response)
            ):
                extra_params['page'] += 1
                results = results + self._get_json(model, space, rel_path, extra_params, get_all=get_all)
            return results
        elif response.status_code == 204:  # No Content
            return []
        else:  # Most likely a 404 Not Found
            raise Exception(
                'Code {0} returned from `{1}`. Response text: "{2}".'.format(
                    response.status_code,
                    url,
                    response.text
                )
            )
1176976
def search(self, query, pagination, result_field):
        
        result = []
        url = "/".join((self.url, query))
        while url:
            log.debug("Pagure query: {0}".format(url))
            try:
                response = requests.get(url, headers=self.headers)
                log.data("Response headers:\n{0}".format(response.headers))
            except requests.RequestException as error:
                log.error(error)
                raise ReportError("Pagure search {0} failed.".format(self.url))
            data = response.json()
            objects = data[result_field]
            log.debug("Result: {0} fetched".format(
                listed(len(objects), "item")))
            log.data(pretty(data))
            # FIXME later: Work around https://pagure.io/pagure/issue/4057
            if not objects:
                break
            result.extend(objects)
            url = data[pagination]['next']
        return result
427699
def get(self, url: str) -> List[dict]:
        
        response = requests.get(
            url,
            headers = {'Content-type': 'application/json'},
            auth = ('scicrunch', 'perl22(query)') # for test2.scicrunch.org
        )
        output = self.process_response(response)
        return output
293358
def get_url_array(self):
        
        urlarray = []
        for urlobjects in self.__json_object["base_urls"]:
            urlarray.append(urlobjects["url"])
        return urlarray
918545
def _get(self, *args, **kwargs):
        """"""
        prev_data = kwargs.pop('prev_data', [])
        resp = self._request('get', *args, **kwargs)
        resp_content = resp._content
        if not resp_content:
            # No content so its obviously not paginated
            return resp

        # if resp._content is a bytes object, decode it so we can loads it as json
        if isinstance(resp_content, bytes):
            resp_content = resp_content.decode('utf-8')

        # Load the json so we can use the data as python objects
        content = json.loads(resp_content)
        if 'pagination' not in content:
            # Result is not paginated
            return resp

        page_info = content['pagination']
        if not page_info['next_uri']:
            # next_uri is None when the cursor has been iterated to the last element
            content['data'].extend(prev_data)
            # If resp._content was is a bytes object, only set it as a bytes object
            if isinstance(resp_content, bytes):
                resp._content = json.dumps(content).decode('utf-8')
            else:
                resp._content = json.dumps(content)
            return resp

        prev_data.extend(content['data'])
        next_page_id = page_info['next_uri'].split('=')[-1]
        kwargs.update({
            'prev_data': prev_data,
            'params': {'starting_after': next_page_id}
        })
        return self._get(*args, **kwargs)
1083375
def get(self, **kwargs):
        
        results = self.search(**kwargs)
        # If there are multiple matches, prefer exact matches on api_version
        if len(results) > 1 and kwargs.get('api_version'):
            results = [
                result for result in results if result.group_version == kwargs['api_version']
            ]
        # If there are multiple matches, prefer non-List kinds
        if len(results) > 1 and not all([isinstance(x, ResourceList) for x in results]):
            results = [result for result in results if not isinstance(result, ResourceList)]
        if len(results) == 1:
            return results[0]
        elif not results:
            raise ResourceNotFoundError('No matches found for {}'.format(kwargs))
        else:
            raise ResourceNotUniqueError('Multiple matches found for {}: {}'.format(kwargs, results))
4589
def google_get_data(self, config, response):
        """"""
        params = {
            'access_token': response['access_token'],
        }
        payload = urlencode(params)
        url = self.google_api_url + 'userinfo?' + payload
        req = Request(url)
        json_str = urlopen(req).read()
        return json.loads(json_str.decode('utf-8'))