Simple requests downloader
438282
def download(url, target_file, chunk_size=4096):
    
    r = requests.get(url, stream=True)

    with open(target_file, 'w+') as out:
        # And this is why I love Armin Ronacher:
        with click.progressbar(r.iter_content(chunk_size=chunk_size),
                               int(r.headers['Content-Length'])/chunk_size,
                               label='Downloading...') as chunks:
            for chunk in chunks:
                out.write(chunk)
30015
def download(url, filename, redownload):
    
    data_folder, filename = filename.rsplit('/', 1)
    download_list = [DownloadRequest(url=url, data_folder=data_folder, filename=filename, save_response=True,
                                     return_data=False)]
    download_data(download_list, redownload=redownload)
7460
function download(source, { verbose, output, onStart, onProgress } = {}) {
  return new Promise(function(y, n) {
    if (typeof output === 'undefined') {
      output = path.basename(url.parse(source).pathname) || 'unknown';
    }

    /**
     * Parse the source url into parts
     */
    const sourceUrl = url.parse(source);

    /**
     * Determine to use https or http request depends on source url
     */
    let request = null;
    if (sourceUrl.protocol === 'https:') {
      request = https.request;
    } else if (sourceUrl.protocol === 'http:') {
      request = http.request;
    } else {
      throw new Error('protocol should be http or https');
    }

    /**
     * Issue the request
     */
    const req = request(
      {
        method: 'GET',
        protocol: sourceUrl.protocol,
        host: sourceUrl.hostname,
        port: sourceUrl.port,
        path: sourceUrl.pathname + (sourceUrl.search || '')
      },
      function(res) {
        if (res.statusCode === 200) {
          const fileSize = Number.isInteger(res.headers['content-length'] - 0)
            ? parseInt(res.headers['content-length'])
            : 0;
          let downloadedSize = 0;

          /**
           * Create write stream
           */
          var writeStream = fs.createWriteStream(output, {
            flags: 'w+',
            encoding: 'binary'
          });

          res.pipe(writeStream);

          /**
           * Invoke `onStartCallback` function
           */
          if (onStart) {
            onStart(res.headers);
          }

          res.on('data', function(chunk) {
            downloadedSize += chunk.length;
            if (onProgress) {
              onProgress({
                fileSize,
                downloadedSize,
                percentage: fileSize > 0 ? downloadedSize / fileSize : 0
              });
            }
          });

          res.on('error', function(err) {
            writeStream.end();
            n(err);
          });

          writeStream.on('finish', function() {
            writeStream.end();
            req.end('finished');
            y({ headers: res.headers, fileSize });
          });
        } else if (
          res.statusCode === 301 ||
          res.statusCode === 302 ||
          res.statusCode === 307
        ) {
          const redirectLocation = res.headers.location;

          if (verbose) {
            console.log('node-wget-promise: Redirected to:', redirectLocation);
          }

          /**
           * Call download function recursively
           */
          download(redirectLocation, {
            output,
            onStart,
            onProgress
          })
            .then(y)
            .catch(n);
        } else {
          n('Server responded with unhandled status: ' + res.statusCode);
        }
      }
    );

    req.end('done');
    req.on('error', err => n(err));
  });
}
887799
def request_fetch(self, user, repo, request, pull=False, force=False): #pragma: no cover
        '''
        '''
        raise NotImplementedError
948121
def download_file(url, filename):
    """"""
    r = _get_requests_session().get(url, stream=True)
    if not r.ok:
        raise IOError("Unable to download file")

    with open(filename, "wb") as f:
        f.write(r.content)
299226
def download(url, filename, overwrite = False):
	'''  '''

	from requests import get
	from os.path import exists

	debug('Downloading ' + unicode(url) + '...')
	data = get(url)
	if data.status_code == 200:
		if not exists(filename) or overwrite:
			f = open(filename, 'wb')
			f.write(data.content)
			f.close()
		return True
	return False
934169
function download(url) {
  return request('GET', url).then(function (res) {
    var out = {
      url: url,
      body: res.body,
      statusCode: res.statusCode,
      headers: res.headers,
      dependencies: []
    };

    try {
      if (out.statusCode === 302) {
        out.dependencies = [out.headers.location];
      } else if (!out.headers['content-type']) {
        out.dependencies = [];
      } else if (out.headers['content-type'].indexOf('text/html') !== -1) {
        out.dependencies = htmldeps(out.body.toString());
      } else if (out.headers['content-type'].indexOf('text/css') !== -1) {
        out.dependencies = cssdeps(out.body.toString());
      } else {
        out.dependencies = [];
      }
    } catch (ex) {
      ex.message = 'Error parsing deps for "' + url + '": ' + ex.message;
      throw ex;
    }
    return out;
  });
}
397156
def _download_file(uri, bulk_api):
    """"""
    resp = requests.get(uri, headers=bulk_api.headers(), stream=True)
    with tempfile.TemporaryFile("w+b") as f:
        for chunk in resp.iter_content(chunk_size=None):
            f.write(chunk)
        f.seek(0)
        yield f
178754
def _request(self, method, endpoint, id=None, **kwargs):
        ""
        request = getattr(requests, method, None)
        if not callable(request):
            raise RequestError('Invalid method %s' % method)
        # Find files, separate them out to correct kwarg for requests.
        data = kwargs.get('data')
        if data:
            files = {}
            for name, value in list(data.items()):
                # Value might be a file-like object (with a read method), or it
                # might be a (filename, file-like) tuple.
                if hasattr(value, 'read') or isinstance(value, tuple):
                    files[name] = data.pop(name)
            if files:
                kwargs.setdefault('files', {}).update(files)
        path = ['api', self.version, endpoint]
        # If we received an ID, append it to the path.
        if id:
            path.append(str(id))
        # Join fragments into a URL
        path = '/'.join(path)
        if not path.endswith('/'):
            path += '/'
        while '//' in path:
            path = path.replace('//', '/')
        url = self.url + path
        # Add our user agent.
        kwargs.setdefault('headers', {}).setdefault('User-Agent',
                                                    HTTP_USER_AGENT)
        # Now try the request, if we get throttled, sleep and try again.
        trys, retrys = 0, 3
        while True:
            if trys == retrys:
                raise RequestError('Could not complete request after %s trys.'
                                   % trys)
            trys += 1
            try:
                return self._do_request(request, url, **kwargs)
            except ResponseError as e:
                if self.throttle_wait and e.status_code == 503:
                    m = THROTTLE_PATTERN.match(
                        e.response.headers.get('x-throttle', ''))
                    if m:
                        time.sleep(float(m.group(1)))
                        continue
                # Failed for a reason other than throttling.
                raise
260815
def _fetch_one(self, request: Request) -> Tuple[bool, float]:
        '''
        '''
        _logger.info(_('Fetching ‘{url}’.'), url=request.url)

        response = None

        try:
            response = yield from self._web_client_session.start()
            self._item_session.response = response

            action = self._result_rule.handle_pre_response(self._item_session)

            if action in (Actions.RETRY, Actions.FINISH):
                raise HookPreResponseBreak()

            self._file_writer_session.process_response(response)

            if not response.body:
                response.body = Body(
                    directory=self._item_session.app_session.root_path,
                    hint='resp_cb'
                )

            yield from \
                self._web_client_session.download(
                    file=response.body,
                    duration_timeout=self._fetch_rule.duration_timeout
                )
        except HookPreResponseBreak:
            _logger.debug('Hook pre-response break.')
            return True, None
        except REMOTE_ERRORS as error:
            self._log_error(request, error)

            self._result_rule.handle_error(self._item_session, error)
            wait_time = self._result_rule.get_wait_time(
                self._item_session, error=error
            )

            if request.body:
                request.body.close()

            if response:
                response.body.close()

            return True, wait_time
        else:
            self._log_response(request, response)
            action = self._handle_response(request, response)
            wait_time = self._result_rule.get_wait_time(self._item_session)

            yield from self._run_coprocessors(request, response)

            response.body.close()

            if request.body:
                request.body.close()

            return action != Actions.NORMAL, wait_time
991904
def fetch_url(url, dest, parent_to_remove_before_fetch):
    """"""
    logger.debug('Downloading file {} from {}', dest, url)
    try:
        shutil.rmtree(parent_to_remove_before_fetch)
    except FileNotFoundError:
        pass
    os.makedirs(parent_to_remove_before_fetch)
    # TODO(itamar): Better downloading (multi-process-multi-threaded?)
    # Consider offloading this to a "standalone app" invoked with Docker
    resp = requests.get(url, stream=True)
    with open(dest, 'wb') as fetch_file:
        for chunk in resp.iter_content(chunk_size=32 * 1024):
            fetch_file.write(chunk)
963983
def download_url(url:str, dest:str, overwrite:bool=False, pbar:ProgressBar=None,
                 show_progress=True, chunk_size=1024*1024, timeout=4, retries=5)->None:
    ""
    if os.path.exists(dest) and not overwrite: return

    s = requests.Session()
    s.mount('http://',requests.adapters.HTTPAdapter(max_retries=retries))
    u = s.get(url, stream=True, timeout=timeout)
    try: file_size = int(u.headers["Content-Length"])
    except: show_progress = False

    with open(dest, 'wb') as f:
        nbytes = 0
        if show_progress: pbar = progress_bar(range(file_size), auto_update=False, leave=False, parent=pbar)
        try:
            for chunk in u.iter_content(chunk_size=chunk_size):
                nbytes += len(chunk)
                if show_progress: pbar.update(nbytes)
                f.write(chunk)
        except requests.exceptions.ConnectionError as e:
            fname = url.split('/')[-1]
            from fastai.datasets import Config
            data_dir = Config().data_path()
            timeout_txt =(f'\n Download of {url} has failed after {retries} retries\n'
                          f' Fix the download manually:\n'
                          f'$ mkdir -p {data_dir}\n'
                          f'$ cd {data_dir}\n'
                          f'$ wget -c {url}\n'
                          f'$ tar -zxvf {fname}\n\n'
                          f'And re-run your code once the download is successful\n')
            print(timeout_txt)
            import sys;sys.exit(1)
565523
function makeHttpRequests(options, fileHashes) {
    if (options.verbose) {
        _utils.logger.info('Performing HTTP requests...');
    }
    var promises = [];
    var _iteratorNormalCompletion4 = true;
    var _didIteratorError4 = false;
    var _iteratorError4 = undefined;

    try {
        for (var _iterator4 = fileHashes[Symbol.iterator](), _step4; !(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion4 = true) {
            var fileWithHash = _step4.value;

            if (fileWithHash.subtitlesPresent) continue;
            if (options.verbose) {
                _utils.logger.info('Downloading subtitles for file [%s] with hash [%s]', fileWithHash.file, fileWithHash.hash);
            }
            var httpRequest = new _utils.HttpRequest(options, fileWithHash);
            promises.push(httpRequest.request());
        }
    } catch (err) {
        _didIteratorError4 = true;
        _iteratorError4 = err;
    } finally {
        try {
            if (!_iteratorNormalCompletion4 && _iterator4.return) {
                _iterator4.return();
            }
        } finally {
            if (_didIteratorError4) {
                throw _iteratorError4;
            }
        }
    }

    return Promise.all(promises);
}
262381
def download(url, file_handle, chunk_size=1024):
    
    r = requests.get(url, stream=True)
    total_length = r.headers.get('content-length')
    if total_length is None:
        maxval = UnknownLength
    else:
        maxval = int(total_length)
    name = file_handle.name
    with progress_bar(name=name, maxval=maxval) as bar:
        for i, chunk in enumerate(r.iter_content(chunk_size)):
            if total_length:
                bar.update(i * chunk_size)
            file_handle.write(chunk)
635133
function downloadFromUrl(page, ph, url, callback) {
    async.waterfall([
        function (callback) {
            page.getCookies(function (cookie) {
                callback(null, page, ph, cookie);
            });
        },
        function (page, ph, cookies, callback) {
            var cookieString = helpers.cookieToHeader(cookies);
            var headers = { Cookie: cookieString };
            request.get({
                url: url,
                headers: headers,
                encoding: null
            }, function (error, response, downloadedBytes) {
                callback(error, page, ph, downloadedBytes);
            });
        }
    ], callback);
}
689417
def http_get(url, fout=None):
    """"""
    print('requests.get({URL}, stream=True)'.format(URL=url))
    rsp = requests.get(url, stream=True)
    if rsp.status_code == 200 and fout is not None:
        with open(fout, 'wb') as prt:
            for chunk in rsp:  # .iter_content(chunk_size=128):
                prt.write(chunk)
            print('  WROTE: {F}\n'.format(F=fout))
    else:
        print(rsp.status_code, rsp.reason, url)
        print(rsp.content)
    return rsp
307482
def get_requests(self):
        
        safe = self.get_safe_struct()

        self.download_list = []
        self.structure_recursion(safe, self.parent_folder)
        self.sort_download_list()
        return self.download_list, self.folder_list