Download cub archive from cub_url and store it in cub_archive_name
979514
def dl_cub(cub_url, cub_archive_name):
    
    with open(cub_archive_name, 'wb') as f:
        remote_file = urllib2.urlopen(cub_url)
        meta = remote_file.info()

        # The server may provide us with the size of the file.
        cl_header = meta.getheaders("Content-Length")
        remote_file_size = int(cl_header[0]) if len(cl_header) > 0 else None

        # Initialise variables
        local_file_size = 0
        block_size = 128*1024

        # Do the download
        while True:
            data = remote_file.read(block_size)

            if not data:
                break

            f.write(data)
            local_file_size += len(data)

        if (remote_file_size is not None and
                not local_file_size == remote_file_size):
            log.warn("Local file size '{}' "
                "does not match remote '{}'".format(
                    local_file_size, remote_file_size))

        remote_file.close()
1102414
def downloadArchiveAction(self, request, queryset):
        '''
        
        '''
        output = io.BytesIO()
        z = zipfile.ZipFile(output, 'w')

        for sub in queryset:
            sub.add_to_zipfile(z)

        z.close()
        # go back to start in ZIP file so that Django can deliver it
        output.seek(0)
        response = HttpResponse(
            output, content_type="application/x-zip-compressed")
        response['Content-Disposition'] = 'attachment; filename=submissions.zip'
        return response
316733
def download_url(url, store_directory, save_name=None, messages=True, suffix=''):
    """"""
    i = url.rfind('/')
    file = url[i+1:]
    print(file)
    dir_name = os.path.join(data_path, store_directory)

    if save_name is None: save_name = os.path.join(dir_name, file)
    else: save_name = os.path.join(dir_name, save_name)

    if suffix is None: suffix=''

    print("Downloading ", url, "->", save_name)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    try:
        response = urlopen(url+suffix)
    except URLError as e:
        if not hasattr(e, "code"):
            raise
        response = e
        if response.code > 399 and response.code<500:
            raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(response.code))
        elif response.code > 499:
            raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(response.code))
    with open(save_name, 'wb') as f:
        meta = response.info()
        content_length_str = meta.get("Content-Length")
        if content_length_str:
            file_size = int(content_length_str)
        else:
            file_size = None
        status = ""
        file_size_dl = 0
        block_sz = 8192
        line_length=30
        while True:
            buff = response.read(block_sz)
            if not buff:
                break
            file_size_dl += len(buff)
            f.write(buff)
            sys.stdout.write(" "*(len(status)) + "\r")
            if file_size:
                status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       full=file_size/(1048576.), ll=line_length,
                                                                       perc="="*int(line_length*float(file_size_dl)/file_size))
            else:
                status = r"[{perc: <{ll}}] {dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       ll=line_length,
                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))

            sys.stdout.write(status)
            sys.stdout.flush()
        sys.stdout.write(" "*(len(status)) + "\r")
        print(status)
1037149
def build_archive(cls, **kwargs):
        
        if cls._archive is None:
            cls._archive = cls(**kwargs)
        return cls._archive
993006
def _save_archive(self):
        
        import json
        from utility import json_serial
        with open(self.archpath, 'w') as f:
            json.dump(self.archive, f, default=json_serial)
493073
def get_data(url, gallery_dir):
    """"""

    # shelve keys need to be str in python 2
    if sys.version_info[0] == 2 and isinstance(url, unicode):
        url = url.encode('utf-8')

    cached_file = os.path.join(gallery_dir, 'searchindex')
    search_index = shelve.open(cached_file)
    if url in search_index:
        data = search_index[url]
    else:
        data = _get_data(url)
        search_index[url] = data
    search_index.close()

    return data
542510
def select(self, binary_request):
    """"""

    logger.debug("binary_request: {!r}".format(binary_request))

    try:
      download_path = self._get_download_path(binary_request)
    except self.MissingMachineInfo as e:
      raise self.BinaryResolutionError(binary_request, e)

    try:
      url_generator = self._get_url_generator(binary_request)
    except self.NoBaseUrlsError as e:
      raise self.BinaryResolutionError(binary_request, e)

    urls = self._get_urls(url_generator, binary_request)
    if not isinstance(urls, list):
      # TODO: add test for this error!
      raise self.BinaryResolutionError(
        binary_request,
        TypeError("urls must be a list: was '{}'.".format(urls)))
    fetch_request = BinaryFetchRequest(
      download_path=download_path,
      urls=urls)

    logger.debug("fetch_request: {!r}".format(fetch_request))

    try:
      downloaded_file = self._binary_tool_fetcher.fetch_binary(fetch_request)
    except BinaryToolFetcher.BinaryNotFound as e:
      raise self.BinaryResolutionError(binary_request, e)

    # NB: we mark the downloaded file executable if it is not an archive.
    archiver = binary_request.archiver
    if archiver is None:
      chmod_plus_x(downloaded_file)
      return downloaded_file

    download_dir = os.path.dirname(downloaded_file)
    # Use the 'name' given in the request as the directory name to extract to.
    unpacked_dirname = os.path.join(download_dir, binary_request.name)
    if not os.path.isdir(unpacked_dirname):
      logger.info("Extracting {} to {} .".format(downloaded_file, unpacked_dirname))
      archiver.extract(downloaded_file, unpacked_dirname, concurrency_safe=True)
    return unpacked_dirname
991904
def fetch_url(url, dest, parent_to_remove_before_fetch):
    """"""
    logger.debug('Downloading file {} from {}', dest, url)
    try:
        shutil.rmtree(parent_to_remove_before_fetch)
    except FileNotFoundError:
        pass
    os.makedirs(parent_to_remove_before_fetch)
    # TODO(itamar): Better downloading (multi-process-multi-threaded?)
    # Consider offloading this to a "standalone app" invoked with Docker
    resp = requests.get(url, stream=True)
    with open(dest, 'wb') as fetch_file:
        for chunk in resp.iter_content(chunk_size=32 * 1024):
            fetch_file.write(chunk)
1343078
def get_from_archive(self, feed_id):
        
        file_path = self.feed_repository + '/' + str(feed_id) + '.xml'
        if not os.path.isfile(file_path):
            raise FeedArchiveNotFound()
        with open(file_path, 'r') as rec_file:
            return rec_file.read()
977562
def get_url(self):
        
        return os.path.join(
            '/archive',
            self.get_year(),
            self.get_month(),
            self.get_day()
        )
878285
def _get_and_write_archive(self, hunt, output_file_path):
    
    hunt_archive = hunt.GetFilesArchive()
    hunt_archive.WriteToFile(output_file_path)
286885
def download_url(url, filename, headers):
    """"""
    ensure_dirs(filename)
    response = requests.get(url, headers=headers, stream=True)
    if response.status_code == 200:
        with open(filename, 'wb') as f:
            for chunk in response.iter_content(16 * 1024):
                f.write(chunk)
5268
def download_url(url, dir_name='.', save_name=None, store_directory=None, messages=True, suffix=''):
    """"""
    if sys.version_info>=(3,0):
        from urllib.parse import quote
        from urllib.request import urlopen 
        from urllib.error import HTTPError, URLError
    else:
        from urllib2 import quote
        from urllib2 import urlopen
        from urllib2 import URLError as HTTPError
    i = url.rfind('/')
    file = url[i+1:]
    if store_directory is not None:
        dir_name = os.path.join(dir_name, store_directory)
    if save_name is None:
        save_name = file
    save_name = os.path.join(dir_name, save_name)
    print("Downloading ", url, "->", save_name)
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    try:
        response = urlopen(url+suffix)
    except HTTPError as e:
        if not hasattr(e, "code"):
            raise
        if e.code > 399 and e.code<500:
            raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(e.code))
        elif e.code > 499:
            raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(e.code))
    except URLError as e:
        raise ValueError('Tried url ' + url + suffix + ' and failed with error ' + str(e.reason))
    with open(save_name, 'wb') as f:
        meta = response.info()
        content_length_str = meta.get("Content-Length")
        if content_length_str:
            #if sys.version_info>=(3,0):
            try:
                file_size = int(content_length_str)
            except:
                try: 
                    file_size = int(content_length_str[0])
                except:
                    file_size = None
            if file_size == 1:
                file_size = None
            #else:
            #    file_size = int(content_length_str)
        else:
            file_size = None
        
        status = ""
        file_size_dl = 0
        block_sz = 8192
        line_length = 30
        percentage = 1./line_length
        
        if file_size:
            print("|"+"{:^{ll}}".format("Downloading {:7.3f}MB".format(file_size/(1048576.)), ll=line_length)+"|")
            from itertools import cycle
            cycle_str = cycle('>')
            sys.stdout.write("|")
                    
        while True:
            buff = response.read(block_sz)
            if not buff:
                break
            file_size_dl += len(buff)
            f.write(buff)
            
            # If content_length_str was incorrect, we can end up with many too many equals signs, catches this edge case
            #correct_meta = float(file_size_dl)/file_size <= 1.0
            
            if file_size:
                if (float(file_size_dl)/file_size) >= percentage:
                    sys.stdout.write(next(cycle_str))
                    sys.stdout.flush()
                    percentage += 1./line_length
                #percentage = "="*int(line_length*float(file_size_dl)/file_size)
                #status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.), full=file_size/(1048576.), ll=line_length, perc=percentage)
            else:
                sys.stdout.write(" "*(len(status)) + "\r")
                status = r"{dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       ll=line_length,
                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))
                sys.stdout.write(status)
                sys.stdout.flush()

            #sys.stdout.write(status)
        
        if file_size:
            sys.stdout.write("|")
            sys.stdout.flush()

        print(status)
1245851
def _download_linkode(self):
        """"""
        # build the API url
        linkode_id = self.url.split("/")[-1]
        if linkode_id.startswith("#"):
            linkode_id = linkode_id[1:]
        url = "https://linkode.org/api/1/linkodes/" + linkode_id

        req = request.Request(url, headers=self.HEADERS_JSON)
        resp = request.urlopen(req)
        raw = resp.read()
        data = json.loads(raw.decode("utf8"))
        content = data['content']
        return content
787977
public function downloadArchiveUrl($options = array(), $nameArchive = null)
    {
        $options = array_merge($options, ['target_public_id' => $nameArchive]);
        return $this->getCloudinary()->download_archive_url($options);
    }
190209
def from_archive(archive_filename, py_interpreter=sys.executable):
    
    with _extract_to_tempdir(archive_filename) as root_dir:
        data = _setup_py_run_from_dir(root_dir, py_interpreter)
    return data
108281
def install_cub(mb_inc_path):
    
    cub_url = 'https://github.com/NVlabs/cub/archive/1.6.4.zip'
    cub_sha_hash = '0d5659200132c2576be0b3959383fa756de6105d'
    cub_version_str = 'Current release: v1.6.4 (12/06/2016)'
    cub_zip_file = 'cub.zip'
    cub_zip_dir = 'cub-1.6.4'
    cub_unzipped_path = os.path.join(mb_inc_path, cub_zip_dir)
    cub_new_unzipped_path = os.path.join(mb_inc_path, 'cub')
    cub_header = os.path.join(cub_new_unzipped_path, 'cub', 'cub.cuh')
    cub_readme = os.path.join(cub_new_unzipped_path, 'README.md' )

    # Check for a reasonably valid install
    cub_installed, _ = is_cub_installed(cub_readme, cub_header, cub_version_str)
    if cub_installed:
        log.info("NVIDIA cub installation found "
            "at '{}'".format(cub_new_unzipped_path))
        return

    log.info("No NVIDIA cub installation found")

    # Do we already have a valid cub zip file
    have_valid_cub_file = (os.path.exists(cub_zip_file) and
        os.path.isfile(cub_zip_file) and
        sha_hash_file(cub_zip_file) == cub_sha_hash)

    if have_valid_cub_file:
        log.info("Valid NVIDIA cub archive found '{}'".format(cub_zip_file))
    # Download if we don't have a valid file
    else:
        log.info("Downloading cub archive '{}'".format(cub_url))
        dl_cub(cub_url, cub_zip_file)
        cub_file_sha_hash = sha_hash_file(cub_zip_file)

        # Compare against our supplied hash
        if cub_sha_hash != cub_file_sha_hash:
            msg = ('Hash of file %s downloaded from %s '
                'is %s and does not match the expected '
                'hash of %s. Please manually download '
                'as per the README.md instructions.') % (
                    cub_zip_file, cub_url,
                    cub_file_sha_hash, cub_sha_hash)

            raise InstallCubException(msg)

    # Unzip into montblanc/include/cub
    with zipfile.ZipFile(cub_zip_file, 'r') as zip_file:
        # Remove any existing installs
        shutil.rmtree(cub_unzipped_path, ignore_errors=True)
        shutil.rmtree(cub_new_unzipped_path, ignore_errors=True)

        # Unzip
        zip_file.extractall(mb_inc_path)

        # Rename. cub_unzipped_path is mb_inc_path/cub_zip_dir
        shutil.move(cub_unzipped_path, cub_new_unzipped_path)

        log.info("NVIDIA cub archive unzipped into '{}'".format(
            cub_new_unzipped_path))


    there, reason = is_cub_installed(cub_readme, cub_header, cub_version_str)

    if not there:
        raise InstallCubException(reason)