make a GET request to freeproxylists.com/elite.html
427339
def _extract_ajax_endpoints(self):
    
        '''  '''
        url = '/'.join([DOC_ROOT, ELITE_PAGE])
        response = requests.get(url)
    
        ''' extract the raw HTML doc from the response '''
        raw_html = response.text
    
        ''' convert raw html into BeautifulSoup object '''
        soup = BeautifulSoup(raw_html, 'lxml')

        for url in soup.select('table tr td table tr td a'):
            if 'elite #' in url.text:
                yield '%s/load_elite_d%s' % (DOC_ROOT, url['href'].lstrip('elite/'))
159867
def scan_cnproxy(self):
        """"""
        self.logger.info(
            'start scanning http://cn-proxy.com for proxy list...')
        response = requests.get('http://cn-proxy.com')
        soup = BeautifulSoup(response.content, 'lxml')
        tables = soup.find_all('table', class_='sortable')
        for table in tables:
            for tr in table.tbody.find_all('tr'):
                info = tr.find_all('td')
                addr = '{}:{}'.format(info[0].string, info[1].string)
                self.proxy_queue.put({'addr': addr, 'protocol': 'http'})
1147391
def _http_get(self, url, query):
        

        if not self.authorization_as_header:
            query.update({'access_token': self.access_token})

        response = None

        self._normalize_query(query)

        kwargs = {
            'params': query,
            'headers': self._request_headers()
        }

        if self._has_proxy():
            kwargs['proxies'] = self._proxy_parameters()

        response = requests.get(
            self._url(url),
            **kwargs
        )

        if response.status_code == 429:
            raise RateLimitExceededError(response)

        return response
1259194
def _get(self, url,
             param_dict={},
             securityHandler=None,
             additional_headers=[],
             handlers=[],
             proxy_url=None,
             proxy_port=None,
             compress=True,
             custom_handlers=[],
             out_folder=None,
             file_name=None):
        
        self._last_method = "GET"
        CHUNK = 4056
        param_dict, handler, cj = self._processHandler(securityHandler, param_dict)
        headers = [] + additional_headers
        if compress:
            headers.append(('Accept-encoding', 'gzip'))
        else:
            headers.append(('Accept-encoding', ''))
        headers.append(('User-Agent', self.useragent))
        if len(param_dict.keys()) == 0:
            param_dict = None
        if handlers is None:
            handlers = []
        if handler is not None:
            handlers.append(handler)
        handlers.append(RedirectHandler())
        if cj is not None:
            handlers.append(request.HTTPCookieProcessor(cj))
        if proxy_url is not None:
            if proxy_port is None:
                proxy_port = 80
            proxies = {"http":"http://%s:%s" % (proxy_url, proxy_port),
                       "https":"https://%s:%s" % (proxy_url, proxy_port)}
            proxy_support = request.ProxyHandler(proxies)
            handlers.append(proxy_support)
        opener = request.build_opener(*handlers)
        opener.addheaders = headers
        if param_dict is None:
            resp = opener.open(url, data=param_dict)
        elif len(str(urlencode(param_dict))) + len(url) >= 1999:
            resp = opener.open(url, data=urlencode(param_dict))
        else:
            format_url = url + "?%s" % urlencode(param_dict)
            resp = opener.open(fullurl=format_url)
        self._last_code = resp.getcode()
        self._last_url = resp.geturl()
        #  Get some headers from the response
        maintype = self._mainType(resp)
        contentDisposition = resp.headers.get('content-disposition')
        contentEncoding = resp.headers.get('content-encoding')
        contentType = resp.headers.get('content-Type').split(';')[0].lower()
        contentLength = resp.headers.get('content-length')
        if maintype.lower() in ('image',
                                'application/x-zip-compressed') or \
           contentType == 'application/x-zip-compressed' or \
           (contentDisposition is not None and \
           contentDisposition.lower().find('attachment;') > -1):

            fname = self._get_file_name(
                contentDisposition=contentDisposition,
                url=url)
            if out_folder is None:
                out_folder = tempfile.gettempdir()
            if contentLength is not None:
                max_length = int(contentLength)
                if max_length < CHUNK:
                    CHUNK = max_length
            file_name = os.path.join(out_folder, fname)
            with open(file_name, 'wb') as writer:
                for data in self._chunk(response=resp,
                                        size=CHUNK):
                    writer.write(data)
                    writer.flush()
                writer.flush()
                del writer
            return file_name
        else:
            read = ""
            for data in self._chunk(response=resp,
                                    size=CHUNK):
                if self.PY3 == True:
                    read += data.decode('utf-8')
                else:
                    read += data

                del data
            try:
                results = json.loads(read)
                if 'error' in results:
                    if 'message' in results['error']:
                        if results['error']['message'] == 'Request not made over ssl':
                            if url.startswith('http://'):
                                url = url.replace('http://', 'https://')
                                return self._get(url,
                                               param_dict,
                                               securityHandler,
                                               additional_headers,
                                               handlers,
                                               proxy_url,
                                               proxy_port,
                                               compress,
                                               custom_handlers,
                                               out_folder,
                                               file_name)
                return results
            except:
                return read
466772
def __request(self, *args, **kwargs):
        

        with self.lock:
            parent = xmlrpc.client.ServerProxy
            # pylint: disable=E1101
            return parent._ServerProxy__request(self, *args, **kwargs)
441756
def get(cls, parent, url_id):
        """"""
        return cls.query.filter_by(parent=parent, url_id=url_id).one_or_none()
801755
def get_proxies(self, proxy_url=None):
        
        query_parameters = {}
        if proxy_url is not None:
            query_parameters['proxyUrl'] = self._serialize.query('proxy_url', proxy_url, 'str')
        response = self._send(http_method='GET',
                              location_id='ec1f4311-f2b4-4c15-b2b8-8990b80d2908',
                              version='5.0-preview.2',
                              query_parameters=query_parameters)
        return self._deserialize('[Proxy]', self._unwrap_collection(response))
852328
def get(query_url,
            headers: {})
      HttpRequest.new(HttpMethodEnum::GET,
                      query_url,
                      headers: headers)
    end
1043028
def get(self, url, params=None, **kwargs):
        """"""
        return requests.get(url, params=params, headers=self.add_headers(**kwargs))
126704
def get(self, request, bot_id, id, format=None):
        
        return super(SourceStateList, self).get(request, bot_id, id, format)
804771
def get_proxies():
    """"""
    proxies = getproxies()
    filtered_proxies = {}
    for key, value in proxies.items():
        if key.startswith('http://'):
            if not value.startswith('http://'):
                filtered_proxies[key] = 'http://{0}'.format(value)
            else:
                filtered_proxies[key] = value
    return filtered_proxies
258741
def get_content(self, uri, disable_proxy=False):
        """"""
        return self.get(uri=uri, disable_proxy=disable_proxy)
1211658
def extractor(url):
    """"""
    response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed)
    if clone:
        mirror(url, response)
    matches = rhref.findall(response)
    for link in matches:
        # Remove everything after a "#" to deal with in-page anchors
        link = link[1].replace('\'', '').replace('"', '').split('#')[0]
        # Checks if the URLs should be crawled
        if is_link(link, processed, files):
            if link[:4] == 'http':
                if link.startswith(main_url):
                    verb('Internal page', link)
                    internal.add(link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:2] == '//':
                if link.split('/')[2].startswith(host):
                    verb('Internal page', link)
                    internal.add(schema + '://' + link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:1] == '/':
                verb('Internal page', link)
                internal.add(remove_file(url) + link)
            else:
                verb('Internal page', link)
                usable_url = remove_file(url)
                if usable_url.endswith('/'):
                    internal.add(usable_url + link)
                elif link.startswith('/'):
                    internal.add(usable_url + link)
                else:
                    internal.add(usable_url + '/' + link)

    if not only_urls:
        intel_extractor(url, response)
        js_extractor(response)
    if args.regex and not supress_regex:
        regxy(args.regex, response, supress_regex, custom)
    if api:
        matches = rentropy.findall(response)
        for match in matches:
            if entropy(match) >= 4:
                verb('Key', match)
                keys.add(url + ': ' + match)
507672
def get(self, uri_relative, params, custom_headers):
        

        return self._request(
            self._METHOD_GET,
            uri_relative,
            self._BYTES_EMPTY,
            params,
            custom_headers
        )
963549
function downloadRouter(sessions, controllers, nonObjTemplatelogLevel, req, res, next) {
    let file = url.parse(req.originalUrl, true).query.file;

    if (req.originalUrl.match(/amorphic\/xhr\?path\=/) && file && req.method === 'GET') {
        processContentRequest(req, res, sessions, controllers, nonObjTemplatelogLevel);
    }
    else {
        next();
    }
}
245081
def requests_get(url):
    

    LOGGER.info(u'Fetching %s…', url)
    return requests.get(url)
65408
def request(self, url, method="GET", params=None, headers=None):
        
        params = params or {}
        headers = headers or {}

        logger.debug("URL: %s", url)
        logger.debug("Method: %s", method)
        logger.debug("Headers: %s", headers)
        logger.debug("Params: %s", params)
        
        response, content = self.client().request(url, method, headers=headers,
            body=urllib.urlencode(params))
        
        content = smart_unicode(content)

        logger.debug("Status: %s", response['status'])
        logger.debug("Content: %s", content)
        
        if response['status'] != '200':
            raise OAuthError(_(
                u"Invalid status code %s while requesting %s: %s") % (
                    response['status'], url, content))
        
        return content