download ERA5 data for GSSHA domain
978497
def _download(self):
        """"""
        # reproject GSSHA grid and get bounds
        min_x, max_x, min_y, max_y = self.gssha_grid.bounds(as_geographic=True)
        if self.era_download_data == 'era5':
            log.info("Downloading ERA5 data ...")
            download_era5_for_gssha(self.lsm_input_folder_path,
                                    self.download_start_datetime,
                                    self.download_end_datetime,
                                    leftlon=min_x-0.5,
                                    rightlon=max_x+0.5,
                                    toplat=max_y+0.5,
                                    bottomlat=min_y-0.5)
        else:
            log.info("Downloading ERA Interim data ...")
            download_interim_for_gssha(self.lsm_input_folder_path,
                                       self.download_start_datetime,
                                       self.download_end_datetime,
                                       leftlon=min_x-1,
                                       rightlon=max_x+1,
                                       toplat=max_y+1,
                                       bottomlat=min_y-1)
637264
def rapid_to_gssha(self):
        
        # if no streamflow given, download forecast
        if self.path_to_rapid_qout is None and self.connection_list_file:
            rapid_qout_directory = os.path.join(self.gssha_directory, 'rapid_streamflow')
            try:
                os.mkdir(rapid_qout_directory)
            except OSError:
                pass
            self.path_to_rapid_qout = self.download_spt_forecast(rapid_qout_directory)

        # prepare input for GSSHA if user wants
        if self.path_to_rapid_qout is not None and self.connection_list_file:
            self.event_manager.prepare_rapid_streamflow(self.path_to_rapid_qout,
                                                        self.connection_list_file)
            self.simulation_modified_input_cards.append('CHAN_POINT_INPUT')
493073
def get_data(url, gallery_dir):
    """"""

    # shelve keys need to be str in python 2
    if sys.version_info[0] == 2 and isinstance(url, unicode):
        url = url.encode('utf-8')

    cached_file = os.path.join(gallery_dir, 'searchindex')
    search_index = shelve.open(cached_file)
    if url in search_index:
        data = search_index[url]
    else:
        data = _get_data(url)
        search_index[url] = data
    search_index.close()

    return data
230090
def download(outformat, path, identifier, namespace='cid', domain='compound', operation=None, searchtype=None,
             overwrite=False, **kwargs):
    """"""
    response = get(identifier, namespace, domain, operation, outformat, searchtype, **kwargs)
    if not overwrite and os.path.isfile(path):
        raise IOError("%s already exists. Use 'overwrite=True' to overwrite it." % path)
    with open(path, 'wb') as f:
        f.write(response)
217326
def get_export(request):
    """"""
    settings = get_current_registry().settings
    exports_dirs = settings['exports-directories'].split()
    args = request.matchdict
    ident_hash, type = args['ident_hash'], args['type']
    id, version = split_ident_hash(ident_hash)

    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            try:
                results = get_export_files(cursor, id, version, [type],
                                           exports_dirs, read_file=True)
                if not results:
                    raise httpexceptions.HTTPNotFound()
                filename, mimetype, size, modtime, state, file_content \
                    = results[0]
            except ExportError as e:
                logger.debug(str(e))
                raise httpexceptions.HTTPNotFound()

    if state == 'missing':
        raise httpexceptions.HTTPNotFound()

    encoded_filename = urllib.quote(filename.encode('utf-8'))
    resp = request.response
    resp.status = "200 OK"
    resp.content_type = mimetype
    #  Need both filename and filename* below for various browsers
    #  See: https://fastmail.blog/2011/06/24/download-non-english-filenames/
    resp.content_disposition = "attachment; filename={fname};" \
                               " filename*=UTF-8''{fname}".format(
                                       fname=encoded_filename)
    resp.body = file_content
    #  Remove version and extension from filename, to recover title slug
    slug_title = '-'.join(encoded_filename.split('-')[:-1])
    resp.headerlist.append(
            ('Link', '<https://{}/contents/{}/{}> ;rel="Canonical"'.format(
                           request.host, id, slug_title)))
    return resp
803012
def get_axfr_records(self, server, domains):
        
        zones = []
        for zoneName in domains:
            try:
                zone = {
                    'zone_id': get_resource_id('axfrz', zoneName),
                    'name': zoneName,
                    'source': 'AXFR',
                    'comment': None,
                    'tags': {},
                    'records': []
                }

                z = dns_zone.from_xfr(query.xfr(server, zoneName))
                rdata_fields = ('name', 'ttl', 'rdata')
                for rr in [dict(zip(rdata_fields, x)) for x in z.iterate_rdatas()]:
                    record_name = rr['name'].derelativize(z.origin).to_text()
                    zone['records'].append(
                    {
                        'id': get_resource_id('axfrr', record_name, ['{}={}'.format(k, str(v)) for k, v in rr.items()]),
                        'zone_id': zone['zone_id'],
                        'name': record_name,
                        'value': sorted([rr['rdata'].to_text()]),
                        'type': type_to_text(rr['rdata'].rdtype)
                    })

                if len(zone['records']) > 0:
                    zones.append(zone)

            except Exception as ex:
                self.log.exception('Failed fetching DNS zone information for {}: {}'.format(zoneName, ex))
                raise

        return zones
232241
def download_urls(self):
    """"""
    urls = {
        tfds.Split.TRAIN: ["train_clean100"],
        tfds.Split.VALIDATION: ["dev_clean"],
        tfds.Split.TEST: ["test_clean"],
    }
    if self.data in ["all", "clean360"]:
      urls[tfds.Split.TRAIN].append("train_clean360")
    if self.data == "all":
      urls[tfds.Split.TRAIN].extend(["train_clean360", "train_other500"])
      urls[tfds.Split.VALIDATION].append("dev_other")
      urls[tfds.Split.TEST].append("test_other")

    urls = {
        split: [_DL_URLS[name] for name in names
               ] for split, names in urls.items()
    }
    return urls
842603
def _domain_event_graphics_cb(conn, domain, phase, local, remote, auth, subject, opaque):
    '''
    
    '''
    prefix = 'VIR_DOMAIN_EVENT_GRAPHICS_'

    def get_address(addr):
        '''
        transform address structure into event data piece
        '''
        return {'family': _get_libvirt_enum_string('{0}_ADDRESS_'.format(prefix), addr['family']),
                'node': addr['node'],
                'service': addr['service']}

    _salt_send_domain_event(opaque, conn, domain, opaque['event'], {
        'phase': _get_libvirt_enum_string(prefix, phase),
        'local': get_address(local),
        'remote': get_address(remote),
        'authScheme': auth,
        'subject': [{'type': item[0], 'name': item[1]} for item in subject]
    })
610222
def download_unicodedata(version, output=HOME, no_zip=False):
    """"""
    files = [
        'UnicodeData.txt',
        'Scripts.txt',
        'Blocks.txt',
        'PropList.txt',
        'DerivedCoreProperties.txt',
        'DerivedNormalizationProps.txt',
        'CompositionExclusions.txt',
        'PropertyValueAliases.txt',
        'PropertyAliases.txt',
        'EastAsianWidth.txt',
        'LineBreak.txt',
        'HangulSyllableType.txt',
        'DerivedAge.txt',
        'auxiliary/WordBreakProperty.txt',
        'auxiliary/SentenceBreakProperty.txt',
        'auxiliary/GraphemeBreakProperty.txt',
        'extracted/DerivedDecompositionType.txt',
        'extracted/DerivedNumericType.txt',
        'extracted/DerivedNumericValues.txt',
        'extracted/DerivedJoiningType.txt',
        'extracted/DerivedJoiningGroup.txt',
        'extracted/DerivedCombiningClass.txt'
    ]

    files.append('ScriptExtensions.txt')
    if PY35:
        files.append('IndicPositionalCategory.txt')
    else:
        files.append('IndicMatraCategory.txt')
    files.append('IndicSyllabicCategory.txt')

    if PY34:
        files.append('BidiBrackets.txt')

    if PY37:
        files.append('VerticalOrientation.txt')

    http_url = 'http://www.unicode.org/Public/%s/ucd/' % version
    ftp_url = 'ftp://ftp.unicode.org/Public/%s/ucd/' % version

    destination = os.path.join(output, 'unicodedata', version)
    if not os.path.exists(destination):
        os.makedirs(destination)

    zip_data = not no_zip

    for f in files:
        file_location = os.path.join(destination, os.path.basename(f))
        retrieved = False
        if not os.path.exists(file_location):
            for url in (ftp_url, http_url):
                furl = url + f
                try:
                    print('Downloading: %s --> %s' % (furl, file_location))
                    response = urlopen(furl, timeout=30)
                    data = response.read()
                except Exception:
                    print('Failed: %s' % url)
                    continue
                with open(file_location, 'w') as uf:
                    uf.write(data.decode('utf-8'))
                retrieved = True
                break
            if not retrieved:
                print('Failed to acquire all needed Unicode files!')
                break
        else:
            retrieved = True
            print('Skipping: found %s' % file_location)

        if not retrieved:
            zip_data = False
            break

    if zip_data and not os.path.exists(os.path.join(output, 'unicodedata', '%s.zip' % version)):
        zip_unicode(output, version)
98860
def _get_data_versions(data):
    
    genome_dir = install.get_genome_dir(data["genome_build"], data["dirs"].get("galaxy"), data)
    if genome_dir:
        version_file = os.path.join(genome_dir, "versions.csv")
        if version_file and os.path.exists(version_file):
            return version_file
    return None
702263
def paths_for_download(self):
        """"""
        if self._paths_for_download is None:
            queries = list()
            try:
                for sra in self.gsm.relations['SRA']:
                    query = sra.split("=")[-1]
                    if 'SRX' not in query:
                        raise ValueError(
                            "Sample looks like it is not an SRA: %s" % query)
                    logger.info("Query: %s" % query)
                    queries.append(query)
            except KeyError:
                raise NoSRARelationException(
                    'No relation called SRA for %s' % self.gsm.get_accession())

            # Construction of DataFrame df with paths to download
            df = DataFrame(columns=['download_path'])
            for query in queries:
                # retrieve IDs for given SRX
                searchdata = Entrez.esearch(db='sra', term=query, usehistory='y',
                                            retmode='json')
                answer = json.loads(searchdata.read())
                ids = answer["esearchresult"]["idlist"]
                if len(ids) != 1:
                    raise ValueError(
                        "There should be one and only one ID per SRX")

                # using ID fetch the info
                number_of_trials = 10
                wait_time = 30
                for trial in range(number_of_trials):
                    try:
                        results = Entrez.efetch(db="sra", id=ids[0],
                                                rettype="runinfo",
                                                retmode="text").read()
                        break
                    except HTTPError as httperr:
                        if "502" in str(httperr):
                            logger.warn(("%s, trial %i out of %i, waiting "
                                         "for %i seconds.") % (
                                             str(httperr),
                                             trial,
                                             number_of_trials,
                                             wait_time))
                            time.sleep(wait_time)
                        elif httperr.code == 429:
                            # This means that there is too many requests
                            try:
                                header_wait_time = int(
                                    httperr.headers["Retry-After"])
                            except:
                                header_wait_time = wait_time
                            logger.warn(("%s, trial %i out of %i, waiting "
                                         "for %i seconds.") % (
                                             str(httperr),
                                             trial,
                                             number_of_trials,
                                             header_wait_time))
                            time.sleep(header_wait_time)
                        else:
                            raise httperr
                try:
                    df_tmp = DataFrame([i.split(',') for i in results.split('\n') if i != ''][1:],
                                       columns=[i.split(',') for i in results.split('\n') if i != ''][0])
                except IndexError:
                    logger.error(("SRA is empty (ID: %s, query: %s). "
                                  "Check if it is publicly available.") %
                                 (ids[0], query))
                    continue

                # check it first
                try:
                    df_tmp['download_path']
                except KeyError as e:
                    logger.error('KeyError: ' + str(e) + '\n')
                    logger.error(str(results) + '\n')

                df = concat([df, df_tmp], sort=True)
            self._paths_for_download = [path for path in df['download_path']]
        return self._paths_for_download
1097611
def srv_rec(rdatas):
    '''
    
    '''
    rschema = OrderedDict((
        ('prio', int),
        ('weight', int),
        ('port', _to_port),
        ('name', str),
    ))
    return _data2rec_group(rschema, rdatas, 'prio')
663476
def download_data(dataset_name=None, prompt=prompt_stdin):
    """"""
        
    dr = data_resources[dataset_name]
    if not authorize_download(dataset_name, prompt=prompt):
        raise Exception("Permission to download data set denied.")
    
    if 'suffices' in dr:
        for url, files, suffices in zip(dr['urls'], dr['files'], dr['suffices']):
            for file, suffix in zip(files, suffices):
                download_url(url=os.path.join(url,file),
                             dir_name = data_path,
                             store_directory=dataset_name,
                             suffix=suffix)
    elif 'dirs' in dr:
        for url, dirs, files in zip(dr['urls'], dr['dirs'], dr['files']):
            for file, dir in zip(files, dirs):
                print(file, dir)
                download_url(
                    url=os.path.join(url,dir,file),
                    dir_name = data_path,
                    store_directory=os.path.join(dataset_name,dir)
                    )
    else:
        for url, files in zip(dr['urls'], dr['files']):
            for file in files:
                download_url(
                    url=os.path.join(url,file),
                    dir_name = data_path,
                    store_directory=dataset_name
                    )
    return True
1144083
def parse_GSE(filepath):
    
    gpls = {}
    gsms = {}
    series_counter = 0
    database = None
    metadata = {}
    gse_name = None
    with utils.smart_open(filepath) as soft:
        groupper = groupby(soft, lambda x: x.startswith("^"))
        for is_new_entry, group in groupper:
            if is_new_entry:
                entry_type, entry_name = __parse_entry(next(group))
                logger.debug("%s: %s" % (entry_type.upper(), entry_name))
                if entry_type == "SERIES":
                    gse_name = entry_name
                    series_counter += 1
                    if series_counter > 1:
                        raise Exception(
                            "GSE file should contain only one series entry!")
                    is_data, data_group = next(groupper)
                    message = ("The key is not False, probably there is an "
                               "error in the SOFT file")
                    assert not is_data, message
                    metadata = parse_metadata(data_group)
                elif entry_type == "SAMPLE":
                    is_data, data_group = next(groupper)
                    gsms[entry_name] = parse_GSM(data_group, entry_name)
                elif entry_type == "PLATFORM":
                    is_data, data_group = next(groupper)
                    gpls[entry_name] = parse_GPL(data_group, entry_name)
                elif entry_type == "DATABASE":
                    is_data, data_group = next(groupper)
                    database_metadata = parse_metadata(data_group)
                    database = GEODatabase(name=entry_name,
                                           metadata=database_metadata)
                else:
                    logger.error("Cannot recognize type %s" % entry_type)
    gse = GSE(name=gse_name,
              metadata=metadata,
              gpls=gpls,
              gsms=gsms,
              database=database)
    return gse
1188639
def _sra_download_worker(*args):
    
    gsm = args[0][0]
    email = args[0][1]
    dirpath = args[0][2]
    kwargs = args[0][3]
    return (gsm.get_accession(), gsm.download_SRA(email, dirpath, **kwargs))
205176
def _get_example_csv(self):
        
        station_key = self.json["station"][0]["key"]
        period = "corrected-archive"
        url = self.url\
                  .replace(".json", "/station/{}/period/{}/data.csv"\
                  .format(station_key, period))

        r = requests.get(url)
        if r.status_code == 200:
            return DataCsv().from_string(r.content)
        else:
            raise Exception("Error connecting to api")
221382
def download_api(branch=None) -> str:
    """"""
    habitica_github_api = 'https://api.github.com/repos/HabitRPG/habitica'
    if not branch:
        branch = requests.get(habitica_github_api + '/releases/latest').json()['tag_name']
    curl = local['curl']['-sL', habitica_github_api + '/tarball/{}'.format(branch)]
    tar = local['tar'][
        'axzf', '-', '--wildcards', '*/website/server/controllers/api-v3/*', '--to-stdout']
    grep = local['grep']['@api']
    sed = local['sed']['-e', 's/^[ */]*//g', '-e', 's/  / /g', '-']
    return (curl | tar | grep | sed)()