Simple curl request made for a given url

    url: str    URL to download
483935
def download_from_url(job, url):
    
    work_dir = job.fileStore.getLocalTempDir()
    file_path = os.path.join(work_dir, os.path.basename(url))
    if not os.path.exists(file_path):
        if url.startswith('s3:'):
            download_from_s3_url(file_path, url)
        else:
            try:
                subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
            except OSError:
                raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
    assert os.path.exists(file_path)
    return job.fileStore.writeGlobalFile(file_path)
963983
def download_url(url:str, dest:str, overwrite:bool=False, pbar:ProgressBar=None,
                 show_progress=True, chunk_size=1024*1024, timeout=4, retries=5)->None:
    ""
    if os.path.exists(dest) and not overwrite: return

    s = requests.Session()
    s.mount('http://',requests.adapters.HTTPAdapter(max_retries=retries))
    u = s.get(url, stream=True, timeout=timeout)
    try: file_size = int(u.headers["Content-Length"])
    except: show_progress = False

    with open(dest, 'wb') as f:
        nbytes = 0
        if show_progress: pbar = progress_bar(range(file_size), auto_update=False, leave=False, parent=pbar)
        try:
            for chunk in u.iter_content(chunk_size=chunk_size):
                nbytes += len(chunk)
                if show_progress: pbar.update(nbytes)
                f.write(chunk)
        except requests.exceptions.ConnectionError as e:
            fname = url.split('/')[-1]
            from fastai.datasets import Config
            data_dir = Config().data_path()
            timeout_txt =(f'\n Download of {url} has failed after {retries} retries\n'
                          f' Fix the download manually:\n'
                          f'$ mkdir -p {data_dir}\n'
                          f'$ cd {data_dir}\n'
                          f'$ wget -c {url}\n'
                          f'$ tar -zxvf {fname}\n\n'
                          f'And re-run your code once the download is successful\n')
            print(timeout_txt)
            import sys;sys.exit(1)
21065
def download_urls(cls, urls, force_download=False):
        
        for url in urls:
            file_path = cls.get_path_to_file_from_url(url)

            if os.path.exists(file_path) and not force_download:
                log.info('already downloaded %s to %s', url, file_path)
            else:
                log.info('downloading %s to %s', url, file_path)
                download_timer = time.time()
                urlretrieve(url, file_path)
                log.info('downloaded in %.2f seconds', time.time() - download_timer)
932821
def _get(url):
    
    if PYTHON_3:
        req = request.Request(url, headers=HEADER)
        response = request.urlopen(req)
        return response.read().decode('utf-8')
    else:
        req = urllib2.Request(url, headers=HEADER)
        response = urllib2.urlopen(req)
        return response.read()
316733
def download_url(url, store_directory, save_name=None, messages=True, suffix=''):
    """"""
    i = url.rfind('/')
    file = url[i+1:]
    print(file)
    dir_name = os.path.join(data_path, store_directory)

    if save_name is None: save_name = os.path.join(dir_name, file)
    else: save_name = os.path.join(dir_name, save_name)

    if suffix is None: suffix=''

    print("Downloading ", url, "->", save_name)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    try:
        response = urlopen(url+suffix)
    except URLError as e:
        if not hasattr(e, "code"):
            raise
        response = e
        if response.code > 399 and response.code<500:
            raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(response.code))
        elif response.code > 499:
            raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(response.code))
    with open(save_name, 'wb') as f:
        meta = response.info()
        content_length_str = meta.get("Content-Length")
        if content_length_str:
            file_size = int(content_length_str)
        else:
            file_size = None
        status = ""
        file_size_dl = 0
        block_sz = 8192
        line_length=30
        while True:
            buff = response.read(block_sz)
            if not buff:
                break
            file_size_dl += len(buff)
            f.write(buff)
            sys.stdout.write(" "*(len(status)) + "\r")
            if file_size:
                status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       full=file_size/(1048576.), ll=line_length,
                                                                       perc="="*int(line_length*float(file_size_dl)/file_size))
            else:
                status = r"[{perc: <{ll}}] {dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       ll=line_length,
                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))

            sys.stdout.write(status)
            sys.stdout.flush()
        sys.stdout.write(" "*(len(status)) + "\r")
        print(status)
1199741
def fromurl(url):
  
  ps = PatchSet( urllib_request.urlopen(url) )
  if ps.errors == 0:
    return ps
  return False
328437
def fetch(
            self,
            url,
            filename=None,
            decompress=False,
            force=False,
            timeout=None,
            use_wget_if_available=True):
        
        key = (url, decompress)
        if not force and key in self._local_paths:
            path = self._local_paths[key]
            if exists(path):
                return path
            else:
                del self._local_paths[key]
        path = download.fetch_file(
            url,
            filename=filename,
            decompress=decompress,
            subdir=self.subdir,
            force=force,
            timeout=timeout,
            use_wget_if_available=use_wget_if_available)

        self._local_paths[key] = path
        return path
659125
def get_req(url):
    """"""
    request = urllib.request.Request(url)
    request.add_header('Authorization', 'token %s' % API_TOKEN)
    try:
        response = urllib.request.urlopen(request).read().decode('utf-8')
        return response
    except urllib.error.HTTPError:
        exception()
        sys.exit(0)
1296147
static function downloadRemoteFile($url, $destinationFilename) {
		if (!ini_get('allow_url_fopen')) {
			throw new \Exception('allow_url_fopen is disable', 501);
		}

        $message = "File was not loaded";

		if (function_exists('curl_init')) {
		    try {
                $raw = file_get_contents($url);
            } catch (\Exception $e) {
                throw new \Exception($message, Consts::ERROR_CODE_BAD_REQUEST);
            }
		} else {
			$ch = curl_init($url);

			curl_setopt($ch, CURLOPT_HEADER, 0);
			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
			curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
			curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
			curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
			curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 30);// таймаут4

			$response = parse_url($url);
			curl_setopt($ch, CURLOPT_REFERER, $response['scheme'] . '://' . $response['host']);
			curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) ' .
                'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.99 YaBrowser/19.1.1.907 Yowser/2.5 Safari/537.36');


			$raw = curl_exec($ch);

            if (!$raw) {
                throw new \Exception($message, Consts::ERROR_CODE_BAD_REQUEST);
            }

            curl_close($ch);
        }

        if ($raw) {
            file_put_contents($destinationFilename, $raw);
        } else {
			throw new \Exception($message, Consts::ERROR_CODE_BAD_REQUEST);
		}
	}
1108388
def download(self, sub_url):
        """"""
        response = requests.get(sub_url, headers=self.headers).text
        soup = BS(response, 'lxml')
        downlink = self.base_url+soup.select('.download a')[0]['href']
        data = requests.get(downlink, headers=self.headers)
        z = zipfile.ZipFile(cStringIO.StringIO(data.content))
        srt_files = [f.filename for f in z.filelist
                     if f.filename.rsplit('.')[-1].lower() in ['srt', 'ass']]
        z.extract(srt_files[0], '/tmp/')
        return srt_files[0]
1081604
def download(url, filename=None):
	
	# requirements os, shutil, urllib.parse, urllib.request
	if not filename:
		url_parts = urllib.parse.urlparse(url)
		filename = os.path.basename(url_parts.path)
	url_h = urllib.request.urlopen(url)
	with open(filename, 'wb') as file_h:
		shutil.copyfileobj(url_h, file_h)
	url_h.close()
	return
725753
def _request(self, url, method="GET", params=None, api_call=None):
        """"""
        method = method.lower()
        params = params or {}
        func = getattr(requests, method)
        requests_args = {}
        if method == "get" or method == "delete":
            requests_args["params"] = params
        else:
            if params.get("json"):
                requests_args["json"] = params.get("json")
            if params.get("files"):
                requests_args["files"] = params.get("files")
            if params.get("data"):
                requests_args["data"] = params.get("data")
        try:
            response = func(url, **requests_args)
        except requests.RequestException as e:
            raise SafecastPyError(str(e))
        # greater than 304 (not modified) is an error
        if response.status_code > 304:
            if response.status_code == 401:
                raise SafecastPyAuthError(response.json().get("error"))
            if response.status_code in [422]:
                raise SafecastPyError(response.json().get("errors"))
            raise SafecastPyError(response.content, error_code=response.status_code)
        try:
            if response.status_code == 204:
                content = response.content
            else:
                content = response.json()
        except ValueError:
            raise SafecastPyError(
                "Response was not valid JSON. \
                               Unable to decode."
            )
        return content
319790
def get_request(self, url):
        
        if isinstance(url, string_types):
            url = urllib2.Request(url, headers={'Accept-encoding': 'identity'})
        return url
966556
def download_file(self, url):
        
        response = requests.get(url, stream=True)
        response.raise_for_status()
        return (int(response.headers.get('content-length', 0)), response)
789411
def download(self, url, path):
        """"""
        # original_url = url
#        print(url)
        qurl = QUrl(url)
        url = to_text_string(qurl.toEncoded(), encoding='utf-8')

        logger.debug(str((url, path)))
        if url in self._workers:
            while not self._workers[url].finished:
                return self._workers[url]

        worker = DownloadWorker(url, path)

        # Check download folder exists
        folder = os.path.dirname(os.path.abspath(path))
        if not os.path.isdir(folder):
            os.makedirs(folder)

        request = QNetworkRequest(qurl)
        self._head_requests[url] = request
        self._paths[url] = path
        self._workers[url] = worker
        self._manager.head(request)
        self._timer.start()

        return worker
282852
function httpGet(url) {
    switch (configuredDownloadClient) {
        case "curl": return sh`curl -A curl -s ${url}`;
        case "wget": return sh`wget -qO- ${url}`;
        case "httpie": return sh`http -b GET ${url}`;
        case "fetch": return sh`fetch -q ${url}`;
    }
}
1339450
private static function curl_get($url)
    {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_TIMEOUT, 60);
        curl_setopt($ch, CURLOPT_HEADER, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);

        $res      = curl_exec($ch);
        $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);

        $header = '';
        $body   = $res;

        if ($httpcode == 200) {
            list($header, $body) = explode("\r\n\r\n", $res, 2);
            $header = Api::http_parse_headers($header);
        }

        $apiReturnData            = [];
        $apiReturnData['type']    = $header['Content-Type'];
        $apiReturnData['size']    = $header['Content-Length'];
        $apiReturnData['header']  = $header;
        $apiReturnData['content'] = $body;

        return $apiReturnData;
    }