Download file, with logic (from tensor2tensor) for Google Drive
335822
def download_from_url(url, path):
    """"""
    def process_response(r):
        chunk_size = 16 * 1024
        total_size = int(r.headers.get('Content-length', 0))
        with open(path, "wb") as file:
            with tqdm(total=total_size, unit='B',
                      unit_scale=1, desc=path.split('/')[-1]) as t:
                for chunk in r.iter_content(chunk_size):
                    if chunk:
                        file.write(chunk)
                        t.update(len(chunk))

    if 'drive.google.com' not in url:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True)
        process_response(response)
        return

    print('downloading from Google Drive; may take a few minutes')
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token
        response = session.get(url, stream=True)

    process_response(response)
623555
function webdriverDownload(done) {
  const platform = os.platform();
  const destDir = path.join(__dirname, '.selenium');
  const chromeDriverUrl = (() => {
    const filePath = platform === 'win32' ?
      '/2.38/chromedriver_win32.zip' :
      `/2.38/chromedriver_${platform === 'darwin' ? 'mac' : 'linux'}64.zip`;
    return `http://chromedriver.storage.googleapis.com${filePath}`;
  })();

  // Only download once.
  if (fs.existsSync(destDir + '/chromedriver') || fs.existsSync(destDir + '/chromedriver.exe')) {
    done();
  } else {
    const selenium = $.download('https://selenium-release.storage.googleapis.com/3.12/selenium-server-standalone-3.12.0.jar')
      .pipe(gulp.dest(destDir));
  
    const chromedriver = $.download(chromeDriverUrl)
      .pipe($.unzip())
      .pipe($.chmod(755))
      .pipe(gulp.dest(destDir));
  
    return mergeStream(selenium, chromedriver);
  }
}
1119094
def download_google_images(path:PathOrStr, search_term:str, size:str='>400*300', n_images:int=10, format:str='jpg',
                            max_workers:int=defaults.cpus, timeout:int=4) -> FilePathList:
    
    label_path = Path(path)/search_term
    search_url = _search_url(search_term, size=size, format=format)
    if n_images <= 100: img_tuples = _fetch_img_tuples(search_url, format=format, n_images=n_images)
    else:               img_tuples = _fetch_img_tuples_webdriver(search_url, format=format, n_images=n_images)
    downloaded_images = _download_images(label_path, img_tuples, max_workers=max_workers, timeout=timeout)
    if len(downloaded_images) == 0: raise RuntimeError(f"Couldn't download any images.")
    verify_images(label_path, max_workers=max_workers)
    return get_image_files(label_path)
1040306
def _augment_file(self, f):
        

        def get_url(target):
            if target.file_size is None:
                return None
            if target.file_name is not None:
                return self.base_url + '/files/content/{0}/{1}'.format(target.file_id.hex, target.file_name)
            else:
                return self.base_url + '/files/content/{0}'.format(target.file_id.hex, )

        f.get_url = types.MethodType(get_url, f)

        def download_to(target, file_name):
            url = target.get_url()
            r = requests.get(url, stream=True)
            with open(file_name, 'wb') as file_to_write:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk:  # filter out keep-alive new chunks
                        file_to_write.write(chunk)
                        file_to_write.flush()
            return file_name

        f.download_to = types.MethodType(download_to, f)
        return f
766259
def download(cls, filename, input_dir, dl_dir=None):
        
        file_info = cls.parse_remote(filename)
        if not dl_dir:
            dl_dir = os.path.join(input_dir, file_info.bucket,
                                  os.path.dirname(file_info.key))
            utils.safe_makedir(dl_dir)

        out_file = os.path.join(dl_dir, os.path.basename(file_info.key))

        if not utils.file_exists(out_file):
            with file_transaction({}, out_file) as tx_out_file:
                command, prog = cls._download_cl(filename)
                if prog == "gof3r":
                    command.extend(["-p", tx_out_file])
                elif prog == "awscli":
                    command.extend([tx_out_file])
                else:
                    raise NotImplementedError(
                        "Unexpected download program %s" % prog)
                subprocess.check_call(command)
        return out_file
1345148
def download_file(client, bucket, remote_path, local_path, overwrite=False):
    """"""
    bucket = client.get_bucket(bucket)
    if (not overwrite) and os.path.isfile(local_path):
        raise OSError('File already exists!')
    with open(local_path, 'wb') as ofh:
        bucket.get_blob(remote_path).download_to_file(ofh)
1105873
def download_file(url, data_path='.', filename=None, size=None, chunk_size=4096, verbose=True):
    """"""
    if filename is None:
        filename = dropbox_basename(url)
    file_path = os.path.join(data_path, filename)
    if url.endswith('?dl=0'):
        url = url[:-1] + '1'  # noninteractive download
    if verbose:
        tqdm_prog = tqdm
        print('requesting URL: {}'.format(url))
    else:
        tqdm_prog = no_tqdm
    r = requests.get(url, stream=True, allow_redirects=True, timeout=5)
    size = r.headers.get('Content-Length', None) if size is None else size
    print('remote size: {}'.format(size))

    stat = path_status(file_path)
    print('local size: {}'.format(stat.get('size', None)))
    if stat['type'] == 'file' and stat['size'] == size:  # TODO: check md5 or get the right size of remote file
        r.close()
        return file_path

    print('Downloading to {}'.format(file_path))

    with open(file_path, 'wb') as f:
        for chunk in r.iter_content(chunk_size=chunk_size):
            if chunk:  # filter out keep-alive chunks
                f.write(chunk)

    r.close()
    return file_path
120977
public void downloadFile(String bucketName, String srcFilename, Path destFilePath)
      throws IOException {
    // [START storage_download_file]
    // The name of the bucket to access
    // String bucketName = "my-bucket";

    // The name of the remote file to download
    // String srcFilename = "file.txt";

    // The path to which the file should be downloaded
    // Path destFilePath = Paths.get("/local/path/to/file.txt");

    // Instantiate a Google Cloud Storage client
    Storage storage = StorageOptions.getDefaultInstance().getService();

    // Get specific file from specified bucket
    Blob blob = storage.get(BlobId.of(bucketName, srcFilename));

    // Download file to specified path
    blob.downloadTo(destFilePath);
    // [END storage_download_file]
  }
861661
def download_from_url(path, url):
  
  filename = url.split("/")[-1]
  found_file = find_file(path, filename, max_depth=0)
  if found_file is None:
    filename = os.path.join(path, filename)
    tf.logging.info("Downloading from %s to %s." % (url, filename))
    inprogress_filepath = filename + ".incomplete"
    inprogress_filepath, _ = urllib.request.urlretrieve(
        url, inprogress_filepath, reporthook=download_report_hook)
    # Print newline to clear the carriage return from the download progress.
    print()
    tf.gfile.Rename(inprogress_filepath, filename)
    return filename
  else:
    tf.logging.info("Already downloaded: %s (at %s)." % (url, found_file))
    return found_file
1006836
def download_to_file(self, file):
        
        con = ConnectionManager().get_connection(self._connection_alias)
        return con.download_to_file(self.file, file, append_base_url=False)
1303424
def download(input, filename, format='sdf', overwrite=False, resolvers=None, **kwargs):
    
    kwargs['format'] = format
    if resolvers:
        kwargs['resolver'] = ",".join(resolvers)
    url = API_BASE+'/%s/file?%s' % (urlquote(input), urlencode(kwargs))
    try:
        servefile = urlopen(url)
        if not overwrite and os.path.isfile(filename):
            raise IOError("%s already exists. Use 'overwrite=True' to overwrite it." % filename)
        file = open(filename, "w")
        file.write(servefile.read())
        file.close()
    except urllib.error.HTTPError:
        # TODO: Proper handling of 404, for now just does nothing
        pass
1187753
def download(odir: Path, source_url: str, irng: Sequence[int]):
    
    assert len(irng) == 2, 'specify start, stop indices'

    odir = Path(odir).expanduser()
    odir.mkdir(parents=True, exist_ok=True)

    ri = int(source_url.split('/')[-2][:2])

    for i in range(*irng):
        fn = f'index-{ri:2d}{i:02d}.fits'
        url = f'{source_url}{fn}'
        ofn = odir / fn
        if ofn.is_file():  # no clobber
            print('skipping', ofn)
            continue
        print(f'{url} => {ofn}', end='\r')

        urlretrieve(url, ofn)
14448
def prepare_data_maybe_download(directory):
    
    filename = 'ubuntu_dialogs.tgz'
    url = 'http://cs.mcgill.ca/~jpineau/datasets/ubuntu-corpus-1.0/ubuntu_dialogs.tgz'
    dialogs_path = os.path.join(directory, 'dialogs')

    # test it there are some dialogs in the path
    if not os.path.exists(os.path.join(directory, "10", "1.tst")):
        # dialogs are missing
        archive_path = os.path.join(directory, filename)
        if not os.path.exists(archive_path):
            # archive missing, download it
            print("Downloading %s to %s" % (url, archive_path))
            filepath, _ = urllib.request.urlretrieve(url, archive_path)
            print "Successfully downloaded " + filepath

        # unpack data
        if not os.path.exists(dialogs_path):
            print("Unpacking dialogs ...")
            with tarfile.open(archive_path) as tar:
                tar.extractall(path=directory)
            print("Archive unpacked.")

        return
158980
def fetch_model(self, source: str, file: Union[str, BinaryIO],
                    chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
        """"""
        download_http(source, file, self._log, chunk_size)
575133
def download(self, force=False, silent=False):
        """"""
        def _download():
            if self.url.startswith("http"):
                self._download_http(silent=silent)
            elif self.url.startswith("ftp"):
                self._download_ftp(silent=silent)
            else:
                raise ValueError("Invalid URL %s" % self.url)
            logger.debug("Moving %s to %s" % (
                self._temp_file_name,
                self.destination))
            shutil.move(self._temp_file_name, self.destination)
            logger.debug("Successfully downloaded %s" % self.url)

        try:
            is_already_downloaded = os.path.isfile(self.destination)
            if is_already_downloaded:
                if force:
                    try:
                        os.remove(self.destination)
                    except Exception:
                        logger.error("Cannot delete %s" % self.destination)
                    logger.info(
                        "Downloading %s to %s" % (self.url, self.destination))
                    logger.debug(
                        "Downloading %s to %s" % (self.url,
                                                  self._temp_file_name))
                    _download()
                else:
                    logger.info(("File %s already exist. Use force=True if you"
                                 " would like to overwrite it.") %
                                self.destination)
            else:
                _download()
        finally:
            try:
                os.remove(self._temp_file_name)
            except OSError:
                pass
179736
def download(self, FILENAME=None):
        """"""
        resultFilename = None
        url = self.__url+'/services'                
        result = Util.retrieveJsonResponseFromServer(url)
        dataItems = result['data']
        for item in dataItems:
            plugin = Plugin(item)        
            result = plugin.getParameterByValue("fr.cnes.sitools.resources.order.DirectOrderResource")            
            if result == None:
                continue
            else:
                urlParameter = plugin.getParameterByType('PARAMETER_ATTACHMENT')
                urlPlugin = urlParameter.getValue()
                encodingParameter = plugin.getParameterByName('archiveType')
                encodingPlugin = encodingParameter.getValue()
                query = self.getQueries()
                query.setBaseUrl(self.__url + urlPlugin)
                url = query.getUrl()                
                (filename, header) = Util.urlretrieve('%s' % url, FILENAME)
                if FILENAME == None:
                    os.rename(filename, filename + "." + encodingPlugin)
                    resultFilename = filename + "." + encodingPlugin
                else:
                    os.rename(FILENAME, FILENAME + "." + encodingPlugin)
                    resultFilename = FILENAME + "." + encodingPlugin
                break
        return resultFilename
901929
def download_from_files(files, output_path, width):
    """"""
    files_to_download = get_files_from_arguments(files, width)
    download_files_if_not_in_manifest(files_to_download, output_path)