Loading time series data from a zip file in the repo
544037
def load_multiformat_time_series():
    """"""
    data = get_example_data('multiformat_time_series.json.gz')
    pdf = pd.read_json(data)

    pdf.ds = pd.to_datetime(pdf.ds, unit='s')
    pdf.ds2 = pd.to_datetime(pdf.ds2, unit='s')
    pdf.to_sql(
        'multiformat_time_series',
        db.engine,
        if_exists='replace',
        chunksize=500,
        dtype={
            'ds': Date,
            'ds2': DateTime,
            'epoch_s': BigInteger,
            'epoch_ms': BigInteger,
            'string0': String(100),
            'string1': String(100),
            'string2': String(100),
            'string3': String(100),
        },
        index=False)
    print('Done loading table!')
    print('-' * 80)
    print('Creating table [multiformat_time_series] reference')
    obj = db.session.query(TBL).filter_by(table_name='multiformat_time_series').first()
    if not obj:
        obj = TBL(table_name='multiformat_time_series')
    obj.main_dttm_col = 'ds'
    obj.database = utils.get_or_create_main_db()
    dttm_and_expr_dict = {
        'ds': [None, None],
        'ds2': [None, None],
        'epoch_s': ['epoch_s', None],
        'epoch_ms': ['epoch_ms', None],
        'string2': ['%Y%m%d-%H%M%S', None],
        'string1': ['%Y-%m-%d^%H:%M:%S', None],
        'string0': ['%Y-%m-%d %H:%M:%S.%f', None],
        'string3': ['%Y/%m/%d%H:%M:%S.%f', None],
    }
    for col in obj.columns:
        dttm_and_expr = dttm_and_expr_dict[col.column_name]
        col.python_date_format = dttm_and_expr[0]
        col.dbatabase_expr = dttm_and_expr[1]
        col.is_dttm = True
    db.session.merge(obj)
    db.session.commit()
    obj.fetch_metadata()
    tbl = obj

    print('Creating Heatmap charts')
    for i, col in enumerate(tbl.columns):
        slice_data = {
            'metrics': ['count'],
            'granularity_sqla': col.column_name,
            'row_limit': config.get('ROW_LIMIT'),
            'since': '2015',
            'until': '2016',
            'where': '',
            'viz_type': 'cal_heatmap',
            'domain_granularity': 'month',
            'subdomain_granularity': 'day',
        }

        slc = Slice(
            slice_name=f'Calendar Heatmap multiformat {i}',
            viz_type='cal_heatmap',
            datasource_type='table',
            datasource_id=tbl.id,
            params=get_slice_json(slice_data),
        )
        merge_slice(slc)
    misc_dash_slices.add('Calendar Heatmap multiformat 0')
269415
def __loaddate():
        ''' 
        '''
        csv_path = os.path.join(os.path.dirname(__file__), 'opendate.csv')
        with open(csv_path) as csv_file:
            csv_data = csv.reader(csv_file)
            result = {}
            result['close'] = []
            result['open'] = []
            for i in csv_data:
                if i[1] == '0':  # 0 = 休市
                    result['close'].append(datetime.strptime(i[0],
                                                             '%Y/%m/%d').date())
                elif i[1] == '1':  # 1 = 開市
                    result['open'].append(datetime.strptime(i[0],
                                                            '%Y/%m/%d').date())
                else:
                    pass
            return result
457697
def load_data_table(file,
                    index_col,
                    show_progress=False):
    
    with ZipFile(file) as zip_file:
        file_names = zip_file.namelist()
        assert len(file_names) == 1, "Expected a single file from Quandl."
        wiki_prices = file_names.pop()
        with zip_file.open(wiki_prices) as table_file:
            if show_progress:
                log.info('Parsing raw data.')
            data_table = pd.read_csv(
                table_file,
                parse_dates=['date'],
                index_col=index_col,
                usecols=[
                    'ticker',
                    'date',
                    'open',
                    'high',
                    'low',
                    'close',
                    'volume',
                    'ex-dividend',
                    'split_ratio',
                ],
            )

    data_table.rename(
        columns={
            'ticker': 'symbol',
            'ex-dividend': 'ex_dividend',
        },
        inplace=True,
        copy=False,
    )
    return data_table
866106
def load_manifest_file(client, bucket, schema, versioned, ifilters, key_info):
    
    # To avoid thundering herd downloads, we do an immediate yield for
    # interspersed i/o
    yield None

    # Inline these values to avoid the local var lookup, they are constants
    # rKey = schema['Key'] # 1
    # rIsLatest = schema['IsLatest'] # 3
    # rVersionId = schema['VersionId'] # 2

    with tempfile.NamedTemporaryFile() as fh:
        client.download_fileobj(Bucket=bucket, Key=key_info['key'], Fileobj=fh)
        fh.seek(0)
        reader = csv.reader(gzip.GzipFile(fileobj=fh, mode='r'))
        for key_set in chunks(reader, 1000):
            keys = []
            for kr in key_set:
                k = kr[1]
                if inventory_filter(ifilters, schema, kr):
                    continue
                k = unquote_plus(k)
                if versioned:
                    if kr[3] == 'true':
                        keys.append((k, kr[2], True))
                    else:
                        keys.append((k, kr[2]))
                else:
                    keys.append(k)
            yield keys
1176085
def load_obs(self):
        
        track_total_file = self.track_data_csv_path + \
            "track_total_{0}_{1}_{2}.csv".format(self.ensemble_name,
                                                 self.ensemble_member,
                                                 self.run_date.strftime("%Y%m%d"))
        track_step_file = self.track_data_csv_path + \
            "track_step_{0}_{1}_{2}.csv".format(self.ensemble_name,
                                                self.ensemble_member,
                                                self.run_date.strftime("%Y%m%d"))
        track_total_cols = ["Track_ID", "Translation_Error_X", "Translation_Error_Y", "Start_Time_Error"]
        track_step_cols = ["Step_ID", "Track_ID", "Hail_Size", "Shape", "Location", "Scale"]
        track_total_data = pd.read_csv(track_total_file, usecols=track_total_cols)
        track_step_data = pd.read_csv(track_step_file, usecols=track_step_cols)
        obs_data = pd.merge(track_step_data, track_total_data, on="Track_ID", how="left")
        self.obs = obs_data
288298
def load_data(self, data):
        

        dates = fill_date_range(self.start_date, self.end_date)
        for row, date in zip(data, dates):
            data = {'date': date}
            if self.add:
                # If self.add is set, results will contain additional
                # attributes (e.g. flags). In that case, create one row per
                # result, with attributes "date", "elem", "value", and one for
                # each item in self.add.
                for elem, vals in zip(self.parameter, row):
                    data['elem'] = elem
                    for add, val in zip(['value'] + self.add, vals):
                        data[add] = val
                    yield data
            else:
                # Otherwise, return one row per date, with "date" and each
                # element's value as attributes.
                for elem, val in zip(self.parameter, row):
                    # namedtuple doesn't like numeric field names
                    if elem.isdigit():
                        elem = "e%s" % elem
                    data[elem] = val
                yield data
21764
def get(tzid):
    """"""
    ns = {}
    path = os.path.join(DATA_DIR, tzid)
    with open(path) as f:
        raw_data = f.read()
    exec(raw_data, ns, ns)
    z = ZoneData()
    z.types = [(delta(offset), delta(save), abbr)
               for offset, save, abbr in ns['types']]
    z.times = [(datetime(*time), i)
               for time, i in ns['times']]
    z.rules = ns['posix']
    return z
822199
def from_zipfile(cls, path, filename, encoding, dialect, fields, converters):
        """"""

        stream = ZipReader(path, filename).readlines(encoding)
        return cls(stream, dialect, fields, converters)
1333064
def datastream_lastmodified(request, pid, dsid, repo=None,
    as_of_date=None, *args, **kwargs):
    '''
    '''
    try:
        if repo is None:
            repo = Repository()
        resp = repo.api.getDatastream(pid, dsid, asOfDateTime=as_of_date)
        dsprofile = parse_xml_object(DatastreamProfile, resp.content, resp.url)
        if dsprofile:
            return dsprofile.created
    except RequestFailed:
        pass
400657
function transform( pkgs, data ) {
	var time;
	var out;
	var len;
	var tmp;
	var pkg;
	var idx;
	var t;
	var v;
	var N;
	var d;
	var i;
	var j;

	len = pkgs.length;

	// Build a time vector based on the query period...
	for ( i = 0; i < len; i++ ) {
		pkg = data[ pkgs[ i ] ];
		if ( pkg ) {
			time = getTime( pkg.start, pkg.end );
			N = time.length;
			break;
		}
	}
	// For each package timeseries, do two things: 1) map each count to a 2-element array format and 2) zero-fill missing values...
	out = new Array( len );
	for ( i = 0; i < len; i++ ) {
		pkg = data[ pkgs[ i ] ];
		if ( pkg === null ) {
			tmp = null;
		} else {
			tmp = new Array( N );
			idx = 0;
			d = pkg.downloads;
			t = ( new Date( d[ idx ].day ) ).getTime();
			for ( j = 0; j < N; j++ ) {
				if ( time[ j ] === t ) {
					v = d[ idx ].downloads;
					idx += 1;

					// Peek ahead to see if we have any more download data for this package...
					if ( d[ idx ] ) {
						t = ( new Date( d[ idx ].day ) ).getTime();
					} else {
						t = 0;
					}
				} else {
					// Zero-fill:
					v = 0;
				}
				tmp[ j ] = [
					(new Date( time[ j ] )).toISOString(),
					v
				];
			}
		}
		out[ i ] = {
			'package': pkgs[ i ],
			'data': tmp
		};
	}
	return out;
}
733923
def obtain_to(filename, compressed=True):
    
    files = glob.glob(filename)
    root, _ = nc.open(filename)
    lat, lon = nc.getvar(root, 'lat')[0,:], nc.getvar(root, 'lon')[0,:]
    date_str = lambda f: ".".join((f.split('/')[-1]).split('.')[1:-2])
    date = lambda f: datetime.strptime(date_str(f), "%Y.%j.%H%M%S")
    linke = lambda f: transform_data(obtain(date(f), compressed), lat, lon)
    linkes_projected = map(lambda f: (f, linke(f)), files)
    nc.close(root)
    return dict(linkes_projected)
1011582
def load_aggregate(args):
    """"""
    repos = load_config(args.config, args.expand_env)
    dirmatch = args.dirmatch
    for repo_dict in repos:
        r = Repo(**repo_dict)
        logger.debug('%s' % r)
        if not match_dir(r.cwd, dirmatch):
            logger.info("Skip %s", r.cwd)
            continue
        r.aggregate()
        if args.do_push:
            r.push()
1066928
def load(filename, drop=False):
    '''
    
    '''
    if filename.startswith('http'):
        log.info('Downloading GeoZones bundle: %s', filename)
        filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))

    log.info('Extracting GeoZones bundle')
    with contextlib.closing(lzma.LZMAFile(filename)) as xz:
        with tarfile.open(fileobj=xz) as f:
            f.extractall(tmp.root)

    log.info('Loading GeoZones levels')

    if drop:
        log.info('Dropping existing levels')
        GeoLevel.drop_collection()

    log.info('Loading levels.msgpack')
    levels_filepath = tmp.path('levels.msgpack')
    with open(levels_filepath) as fp:
        unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
        for i, level in enumerate(unpacker, start=1):
            GeoLevel.objects(id=level['id']).modify(
                upsert=True,
                set__name=level['label'],
                set__parents=[level_ref(p) for p in level['parents']],
                set__admin_level=level.get('admin_level')
            )
    os.remove(levels_filepath)
    log.info('Loaded {total} levels'.format(total=i))

    if drop:
        log.info('Dropping existing spatial zones')
        GeoZone.drop_collection()

    log.info('Loading zones.msgpack')
    zones_filepath = tmp.path('zones.msgpack')
    with open(zones_filepath) as fp:
        unpacker = msgpack.Unpacker(fp, encoding=str('utf-8'))
        unpacker.next()  # Skip headers.
        for i, geozone in enumerate(unpacker):
            params = {
                'slug': slugify.slugify(geozone['name'], separator='-'),
                'level': geozone['level'],
                'code': geozone['code'],
                'name': geozone['name'],
                'keys': geozone.get('keys'),
                'parents': geozone.get('parents', []),
                'ancestors': geozone.get('ancestors', []),
                'successors': geozone.get('successors', []),
                'validity': geozone.get('validity'),
                'population': geozone.get('population'),
                'dbpedia': geozone.get('dbpedia'),
                'flag': geozone.get('flag'),
                'blazon': geozone.get('blazon'),
                'wikipedia': geozone.get('wikipedia'),
                'area': geozone.get('area'),
            }
            if geozone.get('geom') and (
                geozone['geom']['type'] != 'GeometryCollection' or
                    geozone['geom']['geometries']):
                params['geom'] = geozone['geom']
            try:
                GeoZone.objects(id=geozone['_id']).modify(upsert=True, **{
                    'set__{0}'.format(k): v for k, v in params.items()
                })
            except errors.ValidationError as e:
                log.warning('Validation error (%s) for %s with %s',
                            e, geozone['_id'], params)
                continue
    os.remove(zones_filepath)
    log.info('Loaded {total} zones'.format(total=i))

    shutil.rmtree(tmp.path('translations'))
269512
def loadFiles(self, fileNames, rtiClass=None):
        
        for fileName in fileNames:
            self.repo.loadFile(fileName, rtiClass=rtiClass)
1138043
def _load_repodata(filepaths, extra_data=None, metadata=None):
        
        extra_data = extra_data if extra_data else {}
        metadata = metadata if metadata else {}
        repodata = []
        for filepath in filepaths:
            compressed = filepath.endswith('.bz2')
            mode = 'rb' if filepath.endswith('.bz2') else 'r'

            if os.path.isfile(filepath):
                with open(filepath, mode) as f:
                    raw_data = f.read()

                if compressed:
                    data = bz2.decompress(raw_data)
                else:
                    data = raw_data

                try:
                    data = json.loads(to_text_string(data, 'UTF-8'))
                except Exception as error:
                    logger.error(str(error))
                    data = {}

                repodata.append(data)

        all_packages = {}
        for data in repodata:
            packages = data.get('packages', {})
            for canonical_name in packages:
                data = packages[canonical_name]
                name, version, b = tuple(canonical_name.rsplit('-', 2))

                if name not in all_packages:
                    all_packages[name] = {'versions': set(),
                                          'size': {},
                                          'type': {},
                                          'app_entry': {},
                                          'app_type': {},
                                          }
                elif name in metadata:
                    temp_data = all_packages[name]
                    temp_data['home'] = metadata[name].get('home', '')
                    temp_data['license'] = metadata[name].get('license', '')
                    temp_data['summary'] = metadata[name].get('summary', '')
                    temp_data['latest_version'] = metadata[name].get('version')
                    all_packages[name] = temp_data

                all_packages[name]['versions'].add(version)
                all_packages[name]['size'][version] = data.get('size', '')

                # Only the latest builds will have the correct metadata for
                # apps, so only store apps that have the app metadata
                if data.get('type'):
                    all_packages[name]['type'][version] = data.get('type')
                    all_packages[name]['app_entry'][version] = data.get(
                        'app_entry')
                    all_packages[name]['app_type'][version] = data.get(
                        'app_type')

        all_apps = {}
        for name in all_packages:
            versions = sort_versions(list(all_packages[name]['versions']))
            all_packages[name]['versions'] = versions[:]

            for version in versions:
                has_type = all_packages[name].get('type')
                # Has type in this case implies being an app
                if has_type:
                    all_apps[name] = all_packages[name].copy()
                    # Remove all versions that are not apps!
                    versions = all_apps[name]['versions'][:]
                    types = all_apps[name]['type']
                    app_versions = [v for v in versions if v in types]
                    all_apps[name]['versions'] = app_versions

        return all_packages, all_apps
1019815
def _load_timestamps(self):
        """"""
        timestamp_file = os.path.join(self.sequence_path, 'times.txt')

        # Read and parse the timestamps
        self.timestamps = []
        with open(timestamp_file, 'r') as f:
            for line in f.readlines():
                t = dt.timedelta(seconds=float(line))
                self.timestamps.append(t)

        # Subselect the chosen range of frames, if any
        if self.frames is not None:
            self.timestamps = [self.timestamps[i] for i in self.frames]
652377
function(repo, version, hash, meta, outDir) {

    var url, tempRepoDir, packageJSONData, self = this;

    if (meta.vPrefix) {
      version = 'v' + version;
    }

    // Automatically track and cleanup files at exit
    temp.track();

    url = createGitUrl(self.options.baseurl, repo, self.options.reposuffix, self.auth);

    return createTempDir().then(function(tempDir) {
      tempRepoDir = tempDir;
      return exportGitRepo(tempRepoDir, version, url, self.execOpt, self.options.shallowclone);
    }).then(function() {
      return readPackageJSON(tempRepoDir);
    }).then(function(data) {
      packageJSONData = data;
      return moveRepoToOutDir(tempRepoDir, outDir);
    }).then(function() {
      return packageJSONData;
    });
  }