Download a traildb file for a given account/day/region
497258
def download(config, account, day, region, output):
    """"""

    with open(config) as fh:
        config = yaml.safe_load(fh.read())

    jsonschema.validate(config, CONFIG_SCHEMA)

    found = None
    for info in config['accounts']:
        if info['name'] == account:
            found = info
            break

    if not found:
        log.info("Account %s not found", account)
        return

    s3 = boto3.client('s3')
    day = parse_date(day)

    key_data = dict(found)
    key_data['region'] = region
    key_data['date_fmt'] = "%s/%s/%s" % (
        day.year, day.month, day.day)
    key = config['key_template'] % key_data

    s3.download_file(found['bucket'], key, output + '.bz2')
    subprocess.check_call(["lbzip2", "-d", output + '.bz2'])
980594
def download(self, days=60):
        
        days_ago = datetime.datetime.now() - datetime.timedelta(days=days)
        as_of = time.strftime("%Y%m%d", days_ago.timetuple())
        query = self._download_query(as_of=as_of)
        response = self.institution.client().post(query)
        return StringIO(response)
285411
function emitDownload(locale, version, destination) {
	events.emit(eventIds.DOWNLOAD, {locale, version, path: destination});
}
156536
def prepare(self):
        """"""
        if not os.path.exists(self.path):
            if self.retrieve:
                print("Downloading SQLite3 database...")
                download_from_url(self.retrieve, self.path, progress=True)
            else: raise Exception("The file '" + self.path + "' does not exist.")
        self.check_format()
        if self.known_md5: assert self.known_md5 == self.md5
        self.prepared = True
956171
def downloadDatabases(self, savepath=None, unpack=False):
        
        url = self.url('/diagnostics/databases')
        filepath = utils.download(url, self._token, None, savepath, self._session, unpack=unpack)
        return filepath
399472
def fetchFiles(self):
        """"""
        print("Fetching files from {}".format(self.baseDir))
        files = glob(path.join(self.baseDir, '*'))
        filesDef = []
        for f in files:
            print("Found SPARQL file {}".format(f))
            relative = f.replace(self.baseDir, '')
            filesDef.append({
                'download_url': relative,
                'name': relative
            })
        return filesDef
724436
def do_get(self, from_path, to_path):
        
        to_file = open(os.path.expanduser(to_path), "wb")

        self.n.downloadFile(self.current_path + "/" + from_path, to_path)
1174891
def pub_download(args, location=None, prefix=""):
    
    download(location, remotes=args, prefix=prefix)
29790
def fetch(self, is_dl_forced=False):
        ''''''
        cxn = {}
        cxn['host'] = 'nif-db.crbs.ucsd.edu'
        cxn['database'] = 'disco_crawler'
        cxn['port'] = '5432'
        cxn['user'] = config.get_config()['user']['disco']
        cxn['password'] = config.get_config()['keys'][cxn['user']]

        self.dataset.setFileAccessUrl(
            'jdbc:postgresql://'+cxn['host']+':'+cxn['port']+'/'+cxn['database'],
            is_object_literal=True)

        # process the tables
        # self.fetch_from_pgdb(self.tables,cxn,100)  #for testing
        self.fetch_from_pgdb(self.tables, cxn)

        self.get_files(is_dl_forced)

        # FIXME: Everything needed for data provenance?
        fstat = os.stat('/'.join((self.rawdir, 'dvp.pr_nlx_157874_1')))
        filedate = datetime.utcfromtimestamp(fstat[ST_CTIME]).strftime("%Y-%m-%d")
        self.dataset.setVersion(filedate)

        return
1096150
def file_download(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/download' % object_id, input_params, always_retry=always_retry, **kwargs)
775311
def download(user, dl_type, name):
    
    username = user['username']
    user_id = user['id']
    logger.info(
        'Retrieving all {0} of user {1}...'.format(name, username)
    )
    dl_url = url[dl_type].format(user_id)
    logger.debug(dl_url)
    resources = client.get_collection(dl_url, token)
    del resources[:offset - 1]
    logger.debug(resources)
    total = len(resources)
    logger.info('Retrieved {0} {1}'.format(total, name))
    for counter, item in enumerate(resources, offset):
        try:
            logger.debug(item)
            logger.info('{0} n°{1} of {2}'.format(
                name.capitalize(), counter, total)
            )
            if dl_type == 'all':
                item_name = item['type'].split('-')[0]  # remove the '-repost'
                uri = item[item_name]['uri']
                parse_url(uri)
            elif dl_type == 'playlists':
                download_playlist(item)
            elif dl_type == 'playlists-liked':
                parse_url(item['playlist']['uri'])
            elif dl_type == 'commented':
                item = get_track_info(item['track_id'])
                download_track(item)
            else:
                download_track(item)
        except Exception as e:
            logger.exception(e)
    logger.info('Downloaded all {0} {1} of user {2}!'.format(
        total, name, username)
    )
1110734
def download_luts(**kwargs):
    """"""
    #
    import tarfile
    import requests
    TQDM_LOADED = True
    try:
        from tqdm import tqdm
    except ImportError:
        TQDM_LOADED = False

    dry_run = kwargs.get('dry_run', False)

    if 'aerosol_type' in kwargs:
        if isinstance(kwargs['aerosol_type'], (list, tuple, set)):
            aerosol_types = kwargs['aerosol_type']
        else:
            aerosol_types = [kwargs['aerosol_type'], ]
    else:
        aerosol_types = HTTPS_RAYLEIGH_LUTS.keys()

    chunk_size = 10124

    for subname in aerosol_types:

        LOG.debug('Aerosol type: %s', subname)
        http = HTTPS_RAYLEIGH_LUTS[subname]
        LOG.debug('URL = %s', http)

        subdir_path = RAYLEIGH_LUT_DIRS[subname]
        try:
            LOG.debug('Create directory: %s', subdir_path)
            if not dry_run:
                os.makedirs(subdir_path)
        except OSError:
            if not os.path.isdir(subdir_path):
                raise

        if dry_run:
            continue

        response = requests.get(http)
        total_size = int(response.headers['content-length'])

        filename = os.path.join(
            subdir_path, "pyspectral_rayleigh_correction_luts.tgz")
        if TQDM_LOADED:
            with open(filename, "wb") as handle:
                for data in tqdm(iterable=response.iter_content(chunk_size=chunk_size),
                                 total=(total_size / chunk_size), unit='kB'):
                    handle.write(data)
        else:
            with open(filename, "wb") as handle:
                for data in response.iter_content():
                    handle.write(data)

        tar = tarfile.open(filename)
        tar.extractall(subdir_path)
        tar.close()
        os.remove(filename)
1181129
def download(link, outdir='.', chunk_size=4096):
    '''
    
    '''
    url = None
    fh = None
    eta = 'unknown '
    bytes_so_far = 0
    filename = filename_from_url(link) or "."
    cj = cjar.CookieJar()

    # get filename for temp file in current directory
    (fd_tmp, tmpfile) = tempfile.mkstemp(
        ".tmp", prefix=filename + ".", dir=outdir)
    os.close(fd_tmp)
    os.unlink(tmpfile)

    try:
        opener = ulib.build_opener(ulib.HTTPCookieProcessor(cj))
        url = opener.open(link)
        fh = open(tmpfile, mode='wb')

        headers = url.info()
        try:
            total_size = int(headers['Content-Length'])
        except (ValueError, KeyError, TypeError):
            total_size = 'unknown'

        try:
            md5_header = headers['Content-MD5']
        except (ValueError, KeyError, TypeError):
            md5_header = None

        # Define which callback we're gonna use
        if total_size != 'unknown':
            if CONSOLE_WIDTH > 57:
                reporthook = report_bar
            else:
                reporthook = report_onlysize
        else:
            reporthook = report_unknown

        # Below are the registers to calculate network transfer rate
        time_register = time()
        speed = 0.0
        speed_list = []
        bytes_register = 0.0
        eta = 'unknown '

        # Loop that reads in chunks, calculates speed and does the callback to
        # print the progress
        while True:
            chunk = url.read(chunk_size)
            # Update Download Speed every 1 second
            if time() - time_register > 0.5:
                speed = (bytes_so_far - bytes_register) / \
                    (time() - time_register)
                speed_list.append(speed)

                # Set register properly for future use
                time_register = time()
                bytes_register = bytes_so_far

                # Estimative of remaining download time
                if total_size != 'unknown' and len(speed_list) == 3:
                    speed_mean = sum(speed_list) / 3
                    eta_sec = int((total_size - bytes_so_far) / speed_mean)
                    eta = str(datetime.timedelta(seconds=eta_sec))
                    speed_list = []

            bytes_so_far += len(chunk)

            if not chunk:
                sys.stdout.write('\n')
                break

            fh.write(chunk)
            reporthook(bytes_so_far, total_size, speed, eta)
    except KeyboardInterrupt:
        print('\n\nCtrl + C: Download aborted by user')
        print('Partial downloaded file:\n{0}'.format(os.path.abspath(tmpfile)))
        sys.exit(1)
    finally:
        if url:
            url.close()
        if fh:
            fh.close()

    filenamealt = filename_from_headers(headers)
    if filenamealt:
        filename = filenamealt

    # add numeric '(x)' suffix if filename already exists
    if os.path.exists(os.path.join(outdir, filename)):
        filename = filename_fix_existing(filename, outdir)
    filename = os.path.join(outdir, filename)

    shutil.move(tmpfile, filename)

    # Check if sizes matches
    if total_size != 'unknown' and total_size != bytes_so_far:
        print(
            '\n\nWARNING!! Downloaded file size mismatches... Probably corrupted...')

    # Check md5 if it was in html header
    if md5_header:
        print('\nValidating MD5 checksum...')
        if md5_header == md5sum(filename):
            print('MD5 checksum passed!')
        else:
            print('MD5 checksum do NOT passed!!!')

    return filename
1108388
def download(self, sub_url):
        """"""
        response = requests.get(sub_url, headers=self.headers).text
        soup = BS(response, 'lxml')
        downlink = self.base_url+soup.select('.download a')[0]['href']
        data = requests.get(downlink, headers=self.headers)
        z = zipfile.ZipFile(cStringIO.StringIO(data.content))
        srt_files = [f.filename for f in z.filelist
                     if f.filename.rsplit('.')[-1].lower() in ['srt', 'ass']]
        z.extract(srt_files[0], '/tmp/')
        return srt_files[0]
552482
def downloadAndSaveFile( dbid, rid, fid, filename = nil, vid = "0"  )
      response, fileContents = downLoadFile( dbid, rid, fid, vid )
      if fileContents and fileContents.length > 0
         if filename and filename.length > 0
            Misc.save_file( filename, fileContents )
         else
            record = getRecord( rid, dbid, [fid] )
            if record and record[fid] and record[fid].length > 0
               Misc.save_file( record[fid], fileContents )
            else
               Misc.save_file( "#{dbid}_#{rid}_#{fid}", fileContents )
            end
         end
      end
   end
363414
def download_file(fName, time, dire=pDir()):
    
    # hash
    gen_hash = get_hash(fName, 64, dire)
    if gen_hash == -1:
        return -1

    # making request
    user_agent = {'User-agent': 'SubDB/1.0 (sub/0.1; http://github.com/leosartaj/sub)'}
    param = {'action': 'download', 'hash': gen_hash, 'language': 'en'} # Specification for the request
    try:
        r = requests.get("http://api.thesubdb.com/", headers = user_agent, params = param, timeout=time) # Get Request
    except (requests.exceptions.Timeout, socket.error):
        return 'Timeout Error'
    if r.status_code != 200:
        return r.status_code

    # save file
    fName, fExt = os.path.splitext(fName)
    fName += '.srt' # replace extension with srt
    fName = os.path.join(dire, fName) # name coupled with path
    with open(fName, 'wb') as f:
        f.write(r.text.encode('ascii', 'ignore'))

    return r.status_code
1325899
def download(self, uuid, url, forced_file_name=None,
                 progress_bar=True, chunk_size=256,
                 directory=None, overwrite=False):
        
        self.last_req_time = None
        url = self.get_full_url(url)
        self.log.debug("download url : " + url)
        # Building request
        request = urllib2.Request(url)
        # request.add_header('Content-Type', 'application/json; charset=UTF-8')
        request.add_header('Accept', 'application/json,*/*;charset=UTF-8')
        # request start
        starttime = datetime.datetime.now()
        # doRequest
        resultq = urllib2.urlopen(request)
        code = resultq.getcode()
        file_name = uuid
        self.log.debug("ret code : '" + str(code) + "'")
        if code == 200:
            content_lenth = resultq.info().getheader('Content-Length')
            if not content_lenth:
                msg = "No content lengh header found !"
                self.log.debug(msg)
                progress_bar = False
            else:
                file_size = int(content_lenth.strip())
            if forced_file_name:
                file_name = forced_file_name
            else:
                content_dispo = resultq.info().getheader('Content-disposition')
                if content_dispo:
                    content_dispo = content_dispo.strip()
                    file_name = extract_file_name(content_dispo)
            if directory:
                if os.path.isdir(directory):
                    file_name = directory + "/" + file_name
            if os.path.isfile(file_name):
                if not overwrite:
                    cpt = 1
                    while 1:
                        if not os.path.isfile(file_name + "." + str(cpt)):
                            file_name += "." + str(cpt)
                            break
                        cpt += 1
                else:
                    self.log.warn("'%s' already exists. It was overwriten.",
                                  file_name)
            stream = None
            pbar = None
            if progress_bar:
                widgets = [FileTransferSpeed(), ' <<<', Bar(), '>>> ',
                           Percentage(), ' ', ETA()]
                pbar = ProgressBar(widgets=widgets, maxval=file_size)
                stream = FileWithCallback(file_name, 'w', pbar.update,
                                          file_size, file_name)
                pbar.start()
            else:
                stream = file(file_name, 'w')
            while 1:
                chunk = resultq.read(chunk_size)
                if not chunk:
                    break
                stream.write(chunk)
            stream.flush()
            stream.close()
            if pbar:
                pbar.finish()
        # request end
        endtime = datetime.datetime.now()
        self.last_req_time = str(endtime - starttime)
        self.log.debug("download url : %(url)s : request time : %(time)s",
                       {"url": url,
                        "time": self.last_req_time})
        return (file_name, self.last_req_time)