Cleanup temporary files in case of an exception or interrupt.
        This function is not thread-safe.
        :param Downloader self: this
556385
def _cleanup_temporary_files(self):
        # type: (Downloader) -> None
        
        # iterate through dd map and cleanup files
        for key in self._dd_map:
            dd = self._dd_map[key]
            try:
                dd.cleanup_all_temporary_files()
            except Exception as e:
                logger.exception(e)
630541
def download_and_unpack(self, url, *paths, **kw):
        
        with self.temp_download(url, 'ds.zip', log=kw.pop('log', None)) as zipp:
            with TemporaryDirectory() as tmpdir:
                with zipfile.ZipFile(zipp.as_posix()) as zipf:
                    for path in paths:
                        zipf.extract(as_posix(path), path=tmpdir.as_posix())
                        copy(tmpdir.joinpath(path), self)
364478
def download_file(url, path, session=None, params=None):
    

    if url[0:2] == '//':
        url = 'https://' + url[2:]

    # Use a temporary file so that we don't import incomplete files.
    tmp_path = path + '.tmp'

    if session and params:
        r = session.get( url, params=params, stream=True )
    elif session and not params:
        r = session.get( url, stream=True )
    else:
        r = requests.get(url, stream=True)
    with open(tmp_path, 'wb') as f:
        total_length = int(r.headers.get('content-length', 0))
        for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()

    os.rename(tmp_path, path)

    return path
987780
@Override
  public void finish() throws IOException {
    try {
      myLock.writeLock().lock();
      logger.debug("Closing file channel to " + this.current.getName() +
              " (download complete).");
      if (this.channel.isOpen()) {
        this.channel.force(true);
      }

      // Nothing more to do if we're already on the target file.
      if (this.isFinished()) {
        return;
      }

      try {
        FileUtils.deleteQuietly(this.target);
        this.raf.close();
        FileUtils.moveFile(this.current, this.target);
      } catch (Exception ex) {
        logger.error("An error occurred while moving file to its final location", ex);
        if (this.target.exists()) {
          throw new IOException("Was unable to delete existing file " + target.getAbsolutePath(), ex);
        }
        FileUtils.copyFile(this.current, this.target);
      }

      this.current = this.target;

      FileUtils.deleteQuietly(this.partial);
      myIsOpen = false;
      logger.debug("Moved torrent data from {} to {}.",
              this.partial.getName(),
              this.target.getName());
    } finally {
      myLock.writeLock().unlock();
    }
  }
1298101
def _download(self, resource):
    """"""
    if isinstance(resource, six.string_types):
      resource = resource_lib.Resource(url=resource)
    url = resource.url
    if url in self._sizes_checksums:
      expected_sha256 = self._sizes_checksums[url][1]
      download_path = self._get_final_dl_path(url, expected_sha256)
      if not self._force_download and resource.exists_locally(download_path):
        logging.info('URL %s already downloaded: reusing %s.',
                     url, download_path)
        self._recorded_sizes_checksums[url] = self._sizes_checksums[url]
        return promise.Promise.resolve(download_path)
    # There is a slight difference between downloader and extractor here:
    # the extractor manages its own temp directory, while the DownloadManager
    # manages the temp directory of downloader.
    download_dir_path = os.path.join(
        self._download_dir,
        '%s.tmp.%s' % (resource_lib.get_dl_dirname(url), uuid.uuid4().hex))
    tf.io.gfile.makedirs(download_dir_path)
    logging.info('Downloading %s into %s...', url, download_dir_path)

    def callback(val):
      checksum, dl_size = val
      return self._handle_download_result(
          resource, download_dir_path, checksum, dl_size)
    return self._downloader.download(url, download_dir_path).then(callback)
581205
protected final void downloadFile(final URI source, final File destination) throws IOException {
        Validate.validState(!isStopped, "Cannot download file when the crawler is not started.");
        Validate.validState(!isStopping, "Cannot download file when the crawler is stopping.");
        Validate.notNull(source, "The source URL cannot be null.");
        Validate.notNull(destination, "The destination file cannot be null.");

        HttpGet request = new HttpGet(source);
        try (CloseableHttpResponse response = httpClient.execute(request)) {
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                FileUtils.copyInputStreamToFile(entity.getContent(), destination);
            }
        }
    }
270290
public void extract() throws IOException, InterruptedException {

    try {
      extractorProvider.extract(archive, destination, progressListener);
    } catch (IOException ex) {
      try {
        logger.warning("Extraction failed, cleaning up " + destination);
        cleanUp(destination);
      } catch (IOException exx) {
        logger.warning("Failed to cleanup directory");
      }
      // intentional rethrow after cleanup
      throw ex;
    }

    // we do not allow interrupt mid extraction, so catch it here, we still end up
    // with a valid directory though, so don't clean it up.
    if (Thread.currentThread().isInterrupted()) {
      logger.warning("Process was interrupted");
      throw new InterruptedException("Process was interrupted");
    }
  }
1253990
def maybe_download_and_extract(self):
        """"""
        if not os.path.exists(self.cifar10_dir):

            if not os.path.exists(self.data_dir):
                os.makedirs(self.data_dir)

            def _progress(count, block_size, total_size):
                status_msg = '\r>> Downloading {} {:>3}%   '
                sys.stdout.write(status_msg.format(self.cifar10_tarball, float(count * block_size) / total_size * 100.0))
                sys.stdout.flush()

            file_path, _ = urlretrieve(CIFAR10_URL, self.cifar10_tarball, _progress)

            stat_info = os.stat(file_path)
            print('\nSuccessfully downloaded', file_path, stat_info.st_size, 'bytes.\n')

            tarfile.open(file_path, 'r:gz').extractall(self.data_dir)
94566
def maybe_download_and_extract(data_root: str, url: str) -> None:
    

    # make sure data_root exists
    os.makedirs(data_root, exist_ok=True)

    # create sanitized filename from url
    filename = sanitize_url(url)

    # check whether the archive already exists
    filepath = os.path.join(data_root, filename)
    if os.path.exists(filepath):
        logging.info('\t`%s` already exists; skipping', filepath)
        return

    # download with progressbar
    try:
        logging.info('\tdownloading %s', filepath)
        req = requests.get(url, stream=True)
        req.raise_for_status()
    except requests.exceptions.RequestException as ex:
        logging.error('File `%s` could not be downloaded, %s', filepath, ex)
        return

    expected_size = int(req.headers.get('content-length'))
    chunk_size = 1024
    with open(filepath, 'wb') as f_out,\
         click.progressbar(req.iter_content(chunk_size=chunk_size), length=expected_size/chunk_size) as bar:
        for chunk in bar:
            if chunk:
                f_out.write(chunk)
                f_out.flush()

    # extract
    try:
        shutil.unpack_archive(filepath, data_root)
    except (shutil.ReadError, ValueError):
        logging.info('File `%s` could not be extracted by `shutil.unpack_archive`. Please process it manually.',
                     filepath)
163565
static void cleanup() {
        LOGGER.entering();

        for (String temp : files) {
            new File(temp).delete();
        }

        // Cleaning up the files list
        files.clear();
        LOGGER.exiting();
    }
617195
public void downloadToLocalFile(IProgressMonitor monitor, File file)
      throws InvocationTargetException {

    final int taskSize = 1024;

    monitor.setTaskName("Download file " + this.path);

    BufferedOutputStream ostream = null;
    DataInputStream istream = null;

    try {
      istream = getDFS().open(this.path);
      ostream = new BufferedOutputStream(new FileOutputStream(file));

      int bytes;
      byte[] buffer = new byte[taskSize];

      while ((bytes = istream.read(buffer)) >= 0) {
        if (monitor.isCanceled())
          return;
        ostream.write(buffer, 0, bytes);
        monitor.worked(1);
      }

    } catch (Exception e) {
      throw new InvocationTargetException(e);

    } finally {
      // Clean all opened resources
      if (istream != null) {
        try {
          istream.close();
        } catch (IOException e) {
          e.printStackTrace();
          // nothing we can do here
        }
      }
      try {
        ostream.close();
      } catch (IOException e) {
        e.printStackTrace();
        // nothing we can do here
      }
    }
  }
866191
function performCleanup(requester, filesToCache, notifyBack) {
        var currentRequester = requester || "";
        function filterFilesAndNotify(files, filesToCacheArr, notifyBackToBrackets) {
            files.forEach(function (file) {
                var fileExt = path.extname(path.basename(file));
                if (filesToCacheArr.indexOf(fileExt) < 0) {
                    var fileFullPath = updateDir + '/' + file;
                    try {
                        fs.removeSync(fileFullPath);
                    } catch (e) {
                        console.log("AutoUpdate : Exception occured in removing ", fileFullPath, e);
                    }
                }
            });
            if (notifyBackToBrackets) {
                postMessageToBrackets(MessageIds.NOTIFY_SAFE_TO_DOWNLOAD, currentRequester);
            }
        }

        fs.stat(updateDir)
            .then(function (stats) {
                if (stats) {
                    if (filesToCache) {
                        fs.readdir(updateDir)
                            .then(function (files) {
                                filterFilesAndNotify(files, filesToCache, notifyBack);
                            })
                            .catch(function (err) {
                                console.log("AutoUpdate : Error in Reading Update Dir for Cleanup : " + err.toString());
                                postMessageToBrackets(MessageIds.SHOW_ERROR_MESSAGE,
                                                      currentRequester, nodeErrorMessages.UPDATEDIR_READ_FAILED);
                            });
                    } else {
                        fs.remove(updateDir)
                            .then(function () {
                                console.log('AutoUpdate : Update Directory in AppData Cleaned: Complete');
                            })
                            .catch(function (err) {
                                console.log("AutoUpdate : Error in Cleaning Update Dir : " + err.toString());
                                postMessageToBrackets(MessageIds.SHOW_ERROR_MESSAGE,
                                                      currentRequester, nodeErrorMessages.UPDATEDIR_CLEAN_FAILED);
                            });
                    }
                }
            })
            .catch(function (err) {
                console.log("AutoUpdate : Error in Reading Update Dir stats for Cleanup : " + err.toString());
                postMessageToBrackets(MessageIds.SHOW_ERROR_MESSAGE,
                                      currentRequester, nodeErrorMessages.UPDATEDIR_CLEAN_FAILED);
            });
    }
568535
def maybe_download_and_extract():
  """"""
  dest_directory = "/tmp/cifar"
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
    tarfile.open(filepath, 'r:gz').extractall(dest_directory)
158472
def download(self, localdir, max_retry):
        
        local_file = self.gen_localdir(localdir) + self.gen_filename()

        if self.track_exists(localdir):
            print("Track {0} already downloaded, skipping!".format(
                self.get("id")))
            return False

        if local_file in self.get_ignored_tracks(localdir):
            print("\033[93mTrack {0} ignored, skipping!!\033[0m".format(
                self.get("id")))
            return False

        dlurl = self.get_download_link()

        if not dlurl:
            raise serror("Can't download track_id:%d|%s" % (
                self.get("id"),
                self.get("title")))

        retry = max_retry
        print("\nDownloading %s (%d).." % (self.get("title"), self.get("id")))

        while True:
            try:
                urllib.request.urlretrieve(dlurl, local_file,
                                           self._progress_hook)
                break
            except Exception as e:
                if os.path.isfile(local_file):
                    os.unlink(local_file)
                retry -= 1

                if retry < 0:
                    raise serror("Can't download track-id %s, max retry "
                                 "reached (%d). Error occured: %s" % (
                                     self.get("id"), max_retry, type(e)))
                else:
                    print("\033[93mError occured for track-id %s (%s). "
                          "Retrying.. (%d/%d) \033[0m" % (
                              self.get("id"),
                              type(e),
                              max_retry - retry,
                              max_retry))
            except KeyboardInterrupt:
                if os.path.isfile(local_file):
                    os.unlink(local_file)
                raise serror("KeyBoard Interrupt: Incomplete file removed.")

        self.filepath = local_file + self.get_file_extension(local_file)
        os.rename(local_file, self.filepath)
        print("Downloaded => %s" % self.filepath)

        self.downloaded = True
        return True
1159552
def cleanup_temp_filesystem(self):
        
        logger.debug("Deleting compressed file extraction directory: " + self.tmp_dir)
        shutil.rmtree(self.tmp_dir, True)
449255
def finish():
    """"""
    out.warn("Interrupted!")
    for t in threads:
        t.stop()
    jobs.clear()
    out.warn("Waiting for download threads to finish.")
567115
public void shutDown() {
		downloadExecutor.shutdown(); // Disable new tasks from being submitted
		try {
			// Wait a while for existing tasks to terminate
			if (!downloadExecutor.awaitTermination(60, TimeUnit.SECONDS)) {
				downloadExecutor.shutdownNow(); // Cancel currently executing tasks
				// Wait a while for tasks to respond to being cancelled
				if (!downloadExecutor.awaitTermination(60, TimeUnit.SECONDS))
					System.err.println("Pool did not terminate");
			}
		} catch (InterruptedException ie) {
			// (Re-)Cancel if current thread also interrupted
			downloadExecutor.shutdownNow();
			// Preserve interrupt status
			Thread.currentThread().interrupt();
		}
	}