Download cities database.
330464
def download():
    """"""
    url = "http://download.geonames.org/export/dump/cities1000.zip"
    logging.info("Download cities from %s", url)

    if not os.path.exists(MISC_PATH):
        os.makedirs(MISC_PATH)
    zip_path = os.path.join(MISC_PATH, "cities1000.zip")
    urlretrieve(url, zip_path)
    with zipfile.ZipFile(zip_path, "r") as z:
        z.extractall(MISC_PATH)
978497
def _download(self):
        """"""
        # reproject GSSHA grid and get bounds
        min_x, max_x, min_y, max_y = self.gssha_grid.bounds(as_geographic=True)
        if self.era_download_data == 'era5':
            log.info("Downloading ERA5 data ...")
            download_era5_for_gssha(self.lsm_input_folder_path,
                                    self.download_start_datetime,
                                    self.download_end_datetime,
                                    leftlon=min_x-0.5,
                                    rightlon=max_x+0.5,
                                    toplat=max_y+0.5,
                                    bottomlat=min_y-0.5)
        else:
            log.info("Downloading ERA Interim data ...")
            download_interim_for_gssha(self.lsm_input_folder_path,
                                       self.download_start_datetime,
                                       self.download_end_datetime,
                                       leftlon=min_x-1,
                                       rightlon=max_x+1,
                                       toplat=max_y+1,
                                       bottomlat=min_y-1)
618048
def _fetch_data_from_server(raw_data_id, mysql_cfg):
    """"""
    import pymysql
    import pymysql.cursors

    # Import configuration file
    cfg = utils.get_database_configuration()
    if cfg is None:
        return None

    # Establish database connection
    connection = pymysql.connect(host=cfg[mysql_cfg]['host'],
                                 user=cfg[mysql_cfg]['user'],
                                 passwd=cfg[mysql_cfg]['passwd'],
                                 db=cfg[mysql_cfg]['db'],
                                 cursorclass=pymysql.cursors.DictCursor)
    logging.info("Connection: %s", str(connection))
    cursor = connection.cursor()

    # Download dataset
    sql = ("SELECT `id`, `data` "
           "FROM `wm_raw_draw_data` WHERE `id`=%i") % raw_data_id
    cursor.execute(sql)
    return cursor.fetchone()
402497
def download_dataset(tournament, unzip):
    """"""
    click.echo(napi.download_current_dataset(tournament=tournament, unzip=unzip))
1009956
def _download_datasets():
    """"""
    def filepath(*args):
        return abspath(join(dirname(__file__), '..', 'vega_datasets', *args))
    dataset_listing = {}
    for name in DATASETS_TO_DOWNLOAD:
        data = Dataset(name)
        url = data.url
        filename = filepath('_data', data.filename)
        print("retrieving data {0} -> {1}".format(url, filename))
        urlretrieve(url, filename)
        dataset_listing[name] = '_data/{0}'.format(data.filename)
    with open(filepath('local_datasets.json'), 'w') as f:
        json.dump(dataset_listing, f, indent=2, sort_keys=True)
29247
def download_data():
    """"""

    n_datasets = 2
    for d in range(1, n_datasets + 1):
        if not os.path.isdir('enron%d' % d):

            URL = url[d-1]
            print("Downloading %d/%d: %s" % (d, n_datasets, URL))
            folderzip = 'enron%d.zip' % d

            with urlopen(URL) as remotedata:
                with open(folderzip, 'wb') as z:
                    z.write(remotedata.read())

            with ZipFile(folderzip) as z:
                z.extractall()
            os.remove(folderzip)
1360001
def download_database(connection, container: str, target: str=""):
    

    meta_data = objectstore.get_full_container_list(
        connection, container, prefix='database')

    options = return_file_objects(connection, container)

    for o_info in meta_data:
        expected_file = f'database.{ENV}'

        LOG.info(o_info['name'])

        if o_info['name'].startswith(expected_file):
            dt = dateparser.parse(o_info['last_modified'])
            # now = datetime.datetime.now()
            options.append((dt, o_info))

        options.sort()

    if not options:
        LOG.error('Dumps missing? ENVIRONMENT wrong? (acceptance / production')
        LOG.error('Environtment {ENV}')
        sys.exit(1)

    newest = options[-1][1]

    LOG.debug('Downloading: %s', (newest['name']))

    target_file = os.path.join(target, expected_file)

    LOG.info('TARGET: %s', target_file)

    if os.path.exists(target_file):
        LOG.info('Already downloaded')
        return

    LOG.error('TARGET does not exists downloading...')

    new_data = objectstore.get_object(connection, newest, container)

    # save output to file!
    with open(target_file, 'wb') as outputzip:
        outputzip.write(new_data)
453966
def get_city(self):
        
        response = self._session.fetch("users.get", user_ids=self.id, fields="city")[0]
        if response.get('city'):
            return City.from_json(self._session, response.get('city'))
1303643
def get_database(obj, **params):
    """"""
    if isinstance(obj, string_types):
        return connect(obj, **params)
    return obj
190645
def db(self, request):
        ''''''
        with self.mapper.begin() as session:
            world = session.query(World).get(randint(1, 10000))
        return Json(self.get_json(world)).http_response(request)
478949
def get_cities_by_name(self, name):
        

        if name not in self.cities_by_names:
            if self.cities_items is None:
                self.cities_items = list(self.get_cities().items())
            self.cities_by_names[name] = [dict({gid: city})
                for gid, city in self.cities_items if city['name'] == name]
        return self.cities_by_names[name]
844302
def download(client, target_dir):
    """"""
    print('download image previews')
    print(
        "Warning! Downloaded images are only previews!"
        "They may be to small for upload.")
    tree = {}
    listings = client.list('listings')
    languages = map(lambda listing: listing['language'], listings)

    parameters = [{'imageType': image_type, 'language': language}
                  for image_type in image_types for language in languages]
    tree = {image_type: {language: list()
                         for language in languages}
            for image_type in image_types}

    for params in parameters:
        result = client.list('images', **params)
        image_type = params['imageType']
        language = params['language']
        tree[image_type][language] = map(
            lambda r: r['url'], result)

    for image_type, language_map in tree.items():
        for language, files in language_map.items():
            if len(files) > 0:
                mkdir_p(
                    os.path.join(target_dir, 'images', image_type, language))
            if image_type in single_image_types:
                if len(files) > 0:
                    image_url = files[0]
                    path = os.path.join(
                        target_dir,
                        'images',
                        image_type,
                        language,
                        image_type)
                    load_and_save_image(image_url, path)
            else:
                for idx, image_url in enumerate(files):
                    path = os.path.join(
                        target_dir,
                        'images',
                        image_type,
                        language,
                        image_type + '_' + str(idx))
                    load_and_save_image(image_url, path)
232241
def download_urls(self):
    """"""
    urls = {
        tfds.Split.TRAIN: ["train_clean100"],
        tfds.Split.VALIDATION: ["dev_clean"],
        tfds.Split.TEST: ["test_clean"],
    }
    if self.data in ["all", "clean360"]:
      urls[tfds.Split.TRAIN].append("train_clean360")
    if self.data == "all":
      urls[tfds.Split.TRAIN].extend(["train_clean360", "train_other500"])
      urls[tfds.Split.VALIDATION].append("dev_other")
      urls[tfds.Split.TEST].append("test_other")

    urls = {
        split: [_DL_URLS[name] for name in names
               ] for split, names in urls.items()
    }
    return urls
1222492
def pmlr(volumes='all', data_set='pmlr'):
    """"""
    if not data_available(data_set):
        download_data(data_set)
        
    proceedings_file = open(os.path.join(data_path, data_set, 'proceedings.yaml'), 'r')
    import yaml
    proceedings = yaml.load(proceedings_file)
    
    # Create a new resources entry for downloading contents of proceedings.
    data_name_full = 'pmlr_volumes'
    data_resources[data_name_full] = data_resources[data_set].copy()
    data_resources[data_name_full]['files'] = []
    data_resources[data_name_full]['dirs'] = []
    data_resources[data_name_full]['urls'] = []
    for entry in proceedings:
        if volumes=='all' or entry['volume'] in volumes:
            file = entry['yaml'].split('/')[-1]
            dir = 'v' + str(entry['volume'])
            data_resources[data_name_full]['files'].append([file])
            data_resources[data_name_full]['dirs'].append([dir])
            data_resources[data_name_full]['urls'].append(data_resources[data_set]['urls'][0])
    Y = []
    # Download the volume data
    if not data_available(data_name_full):
        download_data(data_name_full)
    for entry in reversed(proceedings):
        volume =  entry['volume']
        if volumes == 'all' or volume in volumes:
            file = entry['yaml'].split('/')[-1]
            volume_file = open(os.path.join(
                data_path, data_name_full,
                'v'+str(volume), file
                ), 'r')
            Y+=yaml.load(volume_file)
    if pandas_available:
        Y = pd.DataFrame(Y)
        Y['published'] = pd.to_datetime(Y['published'])
        #Y.columns.values[4] = json_object('authors')
        #Y.columns.values[7] = json_object('editors')
        Y['issued'] = Y['issued'].apply(lambda x: np.datetime64(datetime.datetime(*x['date-parts'])))
        Y['author'] = Y['author'].apply(lambda x: [str(author['given']) + ' ' + str(author['family']) for author in x])
        Y['editor'] = Y['editor'].apply(lambda x: [str(editor['given']) + ' ' + str(editor['family']) for editor in x])
        columns = list(Y.columns)
        columns[14] = datetime64_('published')
        columns[11] = datetime64_('issued')
        Y.columns = columns
        
    return data_details_return({'Y' : Y, 'info' : 'Data is a pandas data frame containing each paper, its abstract, authors, volumes and venue.'}, data_set)
864428
def load(dirname):
    """"""
    if not os.path.exists(dirname):
        raise AttributeError('No txt database named %s' % dirname)

    db = Database(dirname, dbmode='a')
    chain_folders = [os.path.join(dirname, c) for c in db.get_chains()]
    db.chains = len(chain_folders)

    data = {}
    for chain, folder in enumerate(chain_folders):
        files = os.listdir(folder)
        funnames = funname(files)
        db.trace_names.append(funnames)
        for file in files:
            name = funname(file)
            if name not in data:
                data[
                    name] = {
                }  # This could be simplified using "collections.defaultdict(dict)". New in Python 2.5
            # Read the shape information
            with open(os.path.join(folder, file)) as f:
                f.readline()
                shape = eval(f.readline()[16:])
                data[
                    name][
                        chain] = np.loadtxt(
                            os.path.join(
                                folder,
                                file),
                            delimiter=',').reshape(
                                shape)
                f.close()

    # Create the Traces.
    for name, values in six.iteritems(data):
        db._traces[name] = Trace(name=name, value=values, db=db)
        setattr(db, name, db._traces[name])

    # Load the state.
    statefile = os.path.join(dirname, 'state.txt')
    if os.path.exists(statefile):
        with open(statefile, 'r') as f:
            db._state_ = eval(f.read())
    else:
        db._state_ = {}

    return db
333094
def getDownloadUrls(self):
		""""""
		data = self.searchIndex(False)
		fileUrls = []
		for datum in data:
			fileUrl = self.formatDownloadUrl(datum[0])
			fileUrls.append(fileUrl)
		return fileUrls
1057391
def build_db():
    
    if not os.path.exists(MISC_PATH):
        os.makedirs(MISC_PATH)
    cities_path = os.path.join(MISC_PATH, "cities1000.txt")
    cities_msgpack = os.path.join(MISC_PATH, "cities1000.bin")
    if not os.path.isfile(cities_path):
        download()
    if not os.path.isfile(cities_msgpack):
        with open(cities_path) as f:
            lines = f.readlines()

        db = []
        for i, line in enumerate(lines):
            l = line.strip().split('\t')
            db.append({'latitude': float(l[4]),
                       'longitude': float(l[5]),
                       'linenr': i})
        packed = msgpack.packb(db, use_bin_type=True)
        with open(cities_msgpack, 'wb') as f:
            f.write(packed)
    else:
        with open(cities_msgpack, 'rb') as f:
            content = f.read()
        db = msgpack.unpackb(content)
    return db