Download the cdiff files
198649
def download_diffs(queue):
    """"""
    while True:
        options, signature_type, localver, remotever = queue.get()
        for num in range(int(localver), int(remotever) + 1):
            sig_diff = '%s-%d' % (signature_type, num)
            filename = os.path.join(options.mirrordir, '%s.cdiff' % sig_diff)
            if not os.path.exists(filename):
                update_diff(options, sig_diff)
        queue.task_done()
823434
def download(self, data, block_num=-1):
        
        type_ = c_byte
        size = len(data)
        cdata = (type_ * len(data)).from_buffer_copy(data)
        result = self.library.Cli_Download(self.pointer, block_num,
                                           byref(cdata), size)
        return result
844689
def download(fname, input_dir, dl_dir=None):
    """"""
    try:
        manager = _get_storage_manager(fname)
    except ValueError:
        return fname
    return manager.download(fname, input_dir, dl_dir)
1374356
def cdpop():
    
    if len(_cdhist) >= 1:
        old = _cdhist.pop()  # Pop from history.
        os.chdir(old)
        return old
    else:
        return pwd()
442462
def get_cf_files(path, queue):
    """"""
    for root, _, files in os.walk(os.path.abspath(path)):
        if not files:
            continue
        for filename in files:
            fullname = os.path.join(root, filename)
            if os.path.isfile(fullname) and fullname.endswith('.cf') or \
                    fullname.endswith('.post'):
                queue.put(fullname)
482483
def download(self):
        
        if not os.path.exists(self.data_path):
            # Create if it doesn't exist
            pathlib.Path(self.data_path).mkdir(parents=True, exist_ok=True)

        if not os.path.exists(self.text_path):
            http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())

            with open(self.text_path, 'wt') as fp:
                request = http.request('GET', self.url)
                content = request.data.decode('utf8')
                fp.write(content)

        if not os.path.exists(self.processed_path):
            with open(self.text_path, 'rt') as fp:
                content = fp.read()

            alphabet = sorted(set(content))

            index_to_character = {idx: c for idx, c in enumerate(alphabet, 1)}
            character_to_index = {c: idx for idx, c in enumerate(alphabet, 1)}

            content_encoded = np.array([character_to_index[c] for c in content], dtype=np.uint8)

            data_dict = {
                'alphabet': alphabet,
                'index_to_character': index_to_character,
                'character_to_index': character_to_index,
                'content_encoded': content_encoded
            }

            with open(self.processed_path, 'wb') as fp:
                torch.save(data_dict, fp)
        else:
            with open(self.processed_path, 'rb') as fp:
                data_dict = torch.load(fp)

        return data_dict
542510
def select(self, binary_request):
    """"""

    logger.debug("binary_request: {!r}".format(binary_request))

    try:
      download_path = self._get_download_path(binary_request)
    except self.MissingMachineInfo as e:
      raise self.BinaryResolutionError(binary_request, e)

    try:
      url_generator = self._get_url_generator(binary_request)
    except self.NoBaseUrlsError as e:
      raise self.BinaryResolutionError(binary_request, e)

    urls = self._get_urls(url_generator, binary_request)
    if not isinstance(urls, list):
      # TODO: add test for this error!
      raise self.BinaryResolutionError(
        binary_request,
        TypeError("urls must be a list: was '{}'.".format(urls)))
    fetch_request = BinaryFetchRequest(
      download_path=download_path,
      urls=urls)

    logger.debug("fetch_request: {!r}".format(fetch_request))

    try:
      downloaded_file = self._binary_tool_fetcher.fetch_binary(fetch_request)
    except BinaryToolFetcher.BinaryNotFound as e:
      raise self.BinaryResolutionError(binary_request, e)

    # NB: we mark the downloaded file executable if it is not an archive.
    archiver = binary_request.archiver
    if archiver is None:
      chmod_plus_x(downloaded_file)
      return downloaded_file

    download_dir = os.path.dirname(downloaded_file)
    # Use the 'name' given in the request as the directory name to extract to.
    unpacked_dirname = os.path.join(download_dir, binary_request.name)
    if not os.path.isdir(unpacked_dirname):
      logger.info("Extracting {} to {} .".format(downloaded_file, unpacked_dirname))
      archiver.extract(downloaded_file, unpacked_dirname, concurrency_safe=True)
    return unpacked_dirname
704607
def download_cli(directory, master_token=None, member=None, access_token=None,
                 source=None, project_data=False, max_size='128m',
                 verbose=False, debug=False, memberlist=None,
                 excludelist=None, id_filename=False):
    
    return download(directory, master_token, member, access_token, source,
                    project_data, max_size, verbose, debug, memberlist,
                    excludelist, id_filename)
555195
def _download(self):
        """"""
        print("Downloading!")
        def parse(result):
            print("Got %r back from Yahoo." % (result,))
            values = result.strip().split(",")
            self._value = float(values[1])
        d = getPage(
            "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X"
            % (self._name,))
        d.addCallback(parse)
        d.addErrback(log.err)
        return d
696396
def download(cls, filename, input_dir, dl_dir=None):
        """"""
        file_info = cls.parse_remote(filename)
        if not dl_dir:
            dl_dir = os.path.join(input_dir, file_info.container,
                                  os.path.dirname(file_info.blob))
            utils.safe_makedir(dl_dir)

        out_file = os.path.join(dl_dir, os.path.basename(file_info.blob))

        if not utils.file_exists(out_file):
            with file_transaction({}, out_file) as tx_out_file:
                blob_service = cls.connect(filename)
                blob_service.get_blob_to_path(
                    container_name=file_info.container,
                    blob_name=file_info.blob,
                    file_path=tx_out_file)
        return out_file
1333797
def extract_ace (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    cmdlist = [cmd, 'x']
    if not outdir.endswith('/'):
        outdir += '/'
    cmdlist.extend([archive, outdir])
    return cmdlist
292078
def _download_file_vizier(cat,filePath,catalogname='catalog.dat'):
    '''
    
    '''
    sys.stdout.write('\r'+"Downloading file %s ...\r" \
                         % (os.path.basename(filePath)))
    sys.stdout.flush()
    try:
        # make all intermediate directories
        os.makedirs(os.path.dirname(filePath)) 
    except OSError: pass
    # Safe way of downloading
    downloading= True
    interrupted= False
    file, tmp_savefilename= tempfile.mkstemp()
    os.close(file) #Easier this way
    ntries= 1
    while downloading:
        try:
            ftp= FTP('cdsarc.u-strasbg.fr')
            ftp.login('anonymous', 'test')
            ftp.cwd(os.path.join('pub','cats',cat))
            with open(tmp_savefilename,'wb') as savefile:
                ftp.retrbinary('RETR %s' % catalogname,savefile.write)
            shutil.move(tmp_savefilename,filePath)
            downloading= False
            if interrupted:
                raise KeyboardInterrupt
        except:
            raise
            if not downloading: #Assume KeyboardInterrupt
                raise
            elif ntries > _MAX_NTRIES:
                raise IOError('File %s does not appear to exist on the server ...' % (os.path.basename(filePath)))
        finally:
            if os.path.exists(tmp_savefilename):
                os.remove(tmp_savefilename)
        ntries+= 1
    sys.stdout.write('\r'+_ERASESTR+'\r')
    sys.stdout.flush()        
    return None
1167066
def getfiles(qfiles, dirname, names):
    """"""
    for name in names:
        fullname = os.path.join(dirname, name)
        if os.path.isfile(fullname) and \
            fullname.endswith('.cf') or \
                fullname.endswith('.post'):
            qfiles.put(fullname)
901929
def download_from_files(files, output_path, width):
    """"""
    files_to_download = get_files_from_arguments(files, width)
    download_files_if_not_in_manifest(files_to_download, output_path)
742119
def download(self):
        
        self.downloaded_paths = list()
        for path in self.paths_for_download:
            downloaded_path = list()
            utils.mkdir_p(os.path.abspath(self.directory))

            sra_run = path.split("/")[-1]
            logger.info("Analysing %s" % sra_run)
            url = type(self).FTP_ADDRESS_TPL.format(
                range_subdir=sra_run[:6],
                file_dir=sra_run)
            logger.debug("URL: %s", url)
            filepath = os.path.abspath(
                os.path.join(self.directory, "%s.sra" % sra_run))
            utils.download_from_url(
                url,
                filepath,
                aspera=self.aspera,
                silent=self.silent,
                force=self.force)

            if self.filetype in ("fasta", "fastq"):
                if utils.which('fastq-dump') is None:
                    logger.error("fastq-dump command not found")
                ftype = ""
                if self.filetype == "fasta":
                    ftype = " --fasta "
                cmd = "fastq-dump"
                if utils.which('parallel-fastq-dump') is None:
                    cmd += " %s --outdir %s %s"
                else:
                    logger.debug("Using parallel fastq-dump")
                    cmd = " parallel-fastq-dump --threads %s"
                    cmd = cmd % self.threads
                    cmd += " %s --outdir %s -s %s"
                cmd = cmd % (ftype, self.directory, filepath)

                for fqoption, fqvalue in iteritems(self.fastq_dump_options):
                    if fqvalue:
                        cmd += (" --%s %s" % (fqoption, fqvalue))
                    elif fqvalue is None:
                        cmd += (" --%s" % fqoption)
                logger.debug(cmd)
                process = sp.Popen(cmd, stdout=sp.PIPE,
                                   stderr=sp.PIPE,
                                   shell=True)
                logger.info("Converting to %s/%s*.%s.gz\n" % (
                    self.directory, sra_run, self.filetype))
                pout, perr = process.communicate()
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*.%s.gz" % (sra_run, self.filetype)))

            elif self.filetype == 'sra':
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*.%s" % (sra_run, self.filetype)))

            else:
                downloaded_path = glob.glob(os.path.join(
                    self.directory,
                    "%s*" % sra_run))
                logger.error("Filetype %s not supported." % self.filetype)

            if not self.keep_sra and self.filetype != 'sra':
                # Delete sra file
                os.unlink(filepath)

            self.downloaded_paths += downloaded_path
        return self.downloaded_paths
1018563
def downLoadFile( dbid, rid, fid, vid = "0" )

      @dbid, @rid, @fid, @vid = dbid, rid, fid, vid

      @downLoadFileURL = "http://#{@org}.#{@domain}.com/up/#{dbid}/a/r#{rid}/e#{fid}/v#{vid}"
      
      if @useSSL
         @downLoadFileURL.gsub!( "http:", "https:" )
      end

      @requestHeaders = { "Cookie"  => "ticket=#{@ticket}" }

      if @printRequestsAndResponses
         puts
         puts "downLoadFile request: -------------------------------------"
         p @downLoadFileURL
         p @requestHeaders
      end

      begin

         if USING_HTTPCLIENT
            @responseCode = 404
            @fileContents = @httpConnection.get_content( @downLoadFileURL, nil, @requestHeaders ) 
            @responseCode = 200 if @fileContents
         else  
            @responseCode, @fileContents = @httpConnection.get( @downLoadFileURL, @requestHeaders )
         end
    
      rescue Net::HTTPBadResponse => @lastError
      rescue Net::HTTPHeaderSyntaxError => @lastError
      rescue StandardError => @lastError
      end

      if @printRequestsAndResponses
         puts
         puts "downLoadFile response: -------------------------------------"
         p @responseCode
         p @fileContents
      end

      return self if @chainAPIcalls
      return @responseCode, @fileContents
    end
15887
def find_data(path, warn=True):
    """"""
    full_path = os.path.join(get_data_dir(), path)
    if warn and not os.path.isfile(full_path):
        for package in PACKAGES:
            if path == package.path:
                log.warn('%s doesn\'t exist. Run `cde data download` to get it.' % path)
                break
    return full_path