Fetches metadata for the specified datasources and
        merges to the Superset database
328083
def refresh(self, datasource_names, merge_flag, refreshAll):
        
        session = db.session
        ds_list = (
            session.query(DruidDatasource)
            .filter(DruidDatasource.cluster_name == self.cluster_name)
            .filter(DruidDatasource.datasource_name.in_(datasource_names))
        )
        ds_map = {ds.name: ds for ds in ds_list}
        for ds_name in datasource_names:
            datasource = ds_map.get(ds_name, None)
            if not datasource:
                datasource = DruidDatasource(datasource_name=ds_name)
                with session.no_autoflush:
                    session.add(datasource)
                flasher(
                    _('Adding new datasource [{}]').format(ds_name), 'success')
                ds_map[ds_name] = datasource
            elif refreshAll:
                flasher(
                    _('Refreshing datasource [{}]').format(ds_name), 'info')
            else:
                del ds_map[ds_name]
                continue
            datasource.cluster = self
            datasource.merge_flag = merge_flag
        session.flush()

        # Prepare multithreaded executation
        pool = ThreadPool()
        ds_refresh = list(ds_map.values())
        metadata = pool.map(_fetch_metadata_for, ds_refresh)
        pool.close()
        pool.join()

        for i in range(0, len(ds_refresh)):
            datasource = ds_refresh[i]
            cols = metadata[i]
            if cols:
                col_objs_list = (
                    session.query(DruidColumn)
                    .filter(DruidColumn.datasource_id == datasource.id)
                    .filter(DruidColumn.column_name.in_(cols.keys()))
                )
                col_objs = {col.column_name: col for col in col_objs_list}
                for col in cols:
                    if col == '__time':  # skip the time column
                        continue
                    col_obj = col_objs.get(col)
                    if not col_obj:
                        col_obj = DruidColumn(
                            datasource_id=datasource.id,
                            column_name=col)
                        with session.no_autoflush:
                            session.add(col_obj)
                    col_obj.type = cols[col]['type']
                    col_obj.datasource = datasource
                    if col_obj.type == 'STRING':
                        col_obj.groupby = True
                        col_obj.filterable = True
                datasource.refresh_metrics()
        session.commit()
33325
def load_from_dict(db, data, metadata):
    
    # The ORM overhead is too high for this kind of bulk load,
    # so drop down to SQLAlchemy Core.
    # This tries to do minimize the number of SQL commands by loading entire
    # tables at once

    if metadata['version'] != 2:
        raise ValueError('Can only load version 2')

    try:
        bind = db.get_bind(tables.Event)
    except (ValueError, AttributeError):
        pass
    else:
        if bind.dialect.name == 'sqlite':
            db.execute('PRAGMA foreign_keys = ON')

    with bulk_inserter(db) as insert:

        # Load speakers

        speaker_slugs = set()

        for series_slug, series in data['series'].items():
            for event_slug, event in series['events'].items():
                for talk in event.get('talks'):
                    for speaker in talk.get('speakers', ()):
                        if speaker not in speaker_slugs:
                            speaker_slugs.add(speaker)
                            insert(tables.Speaker, {
                                'slug': speaker,
                                'name': speaker,
                            })

        venue_ids = {}

        # Load cities, and their venues

        for city_slug, city in data['cities'].items():
            city_data = city['city']
            insert(tables.City, {
                'slug': city_slug,
                'name': city_data['name'],
                'latitude': city_data['location']['latitude'],
                'longitude': city_data['location']['longitude'],
                '_source': city_data['_source'],
            })

            for venue_slug, venue in city.get('venues', {}).items():
                venue_ids[city_slug, venue_slug] = insert(tables.Venue, {
                    'city_slug': city_slug,
                    'slug': venue_slug,
                    'name': venue['name'],
                    'address': venue.get('address'),
                    'latitude': venue['location']['latitude'],
                    'longitude': venue['location']['longitude'],
                    'notes': venue.get('notes'),
                })


        # Load series, their events, and everything underneath

        for series_slug, series_dir in data['series'].items():

            series = series_dir['series']
            recurrence = series.get('recurrence')
            if recurrence:
                rrule_str = recurrence['rrule']
                rrule.rrulestr(rrule_str)  # check rrule syntax
                recurrence_attrs = {
                    'recurrence_rule': rrule_str,
                    'recurrence_scheme': recurrence['scheme'],
                    'recurrence_description_cs': recurrence['description']['cs'],
                    'recurrence_description_en': recurrence['description']['en'],
                }
            else:
                recurrence_attrs = {
                    'recurrence_rule': None,
                    'recurrence_scheme': None,
                    'recurrence_description_cs': None,
                    'recurrence_description_en': None,
                }
            insert(tables.Series, {
                'slug': series_slug,
                'name': series['name'],
                'home_city_slug': series.get('city'),
                'description_cs': series['description']['cs'],
                'description_en': series['description']['en'],
                'organizer_info': json.dumps(series.get('organizer-info', ())),
                **recurrence_attrs,
            })

            for event_slug, event in series_dir['events'].items():
                venue_slug = event.get('venue')
                city_slug = event['city']
                if venue_slug:
                    venue_id = venue_ids[city_slug, venue_slug]
                else:
                    venue_id = None

                start = make_full_datetime(event['start'])
                end = event.get('end')
                if end is None:
                    end = start.replace(hour=23, minute=59, second=59)
                event_id = insert(tables.Event, {
                    'name': event['name'],
                    'number': event.get('number'),
                    'topic': event.get('topic'),
                    'description': event.get('description'),
                    'date': start.date(),
                    'start_time': start.time(),
                    'end': end,
                    'all_day': event.get('all_day', False),
                    'series_slug': series_slug,
                    'city_slug': city_slug,
                    'venue_id': venue_id,
                    '_source': event['_source']
                })

                for i, talk in enumerate(event.get('talks', ())):
                    talk_id = insert(tables.Talk, {
                        'event_id': event_id,
                        'index': i,
                        'title': talk['title'],
                        'description': talk.get('description'),
                        'is_lightning': talk.get('lightning', False),
                    })

                    for i, speaker in enumerate(talk.get('speakers', ())):
                        assert speaker in speaker_slugs
                        insert(tables.TalkSpeaker, {
                            'talk_id': talk_id,
                            'index': i,
                            'speaker_slug': speaker,
                        })

                    for i, link in enumerate([
                            *({'talk': u} for u in talk.get('urls', ())),
                            *talk.get('coverage', {})]):
                        for kind, url in link.items():
                            insert(tables.TalkLink, {
                                'talk_id': talk_id,
                                'index': i,
                                'url': url,
                                'kind': kind,
                            })

                for i, url in enumerate(event.get('urls', ())):
                    insert(tables.EventLink, {
                        'event_id': event_id,
                        'index': i,
                        'url': url,
                    })
1121784
public static DataSource getDataSource(String source) {
		DataSource ds = dataSources.get(source);
		if (ds == null){
			synchronized (dataSourcesStructureLock){
				ds = dataSources.get(source);
				if (ds == null){
					ds = createDataSource(source);
					if (ds != null){
						dataSources.put(source, ds);
					}
				}
			}
		}
		return ds;
	}
289221
def _GetMetadataForClients(self, client_urns):
    """"""

    result = {}
    metadata_to_fetch = set()

    for urn in client_urns:
      try:
        result[urn] = self._cached_metadata[urn]
      except KeyError:
        metadata_to_fetch.add(urn)

    if metadata_to_fetch:
      if data_store.RelationalDBEnabled():
        client_ids = set(urn.Basename() for urn in metadata_to_fetch)
        infos = data_store.REL_DB.MultiReadClientFullInfo(client_ids)

        fetched_metadata = [
            export.GetMetadata(client_id, info)
            for client_id, info in infos.items()
        ]
      else:
        client_fds = aff4.FACTORY.MultiOpen(
            metadata_to_fetch, mode="r", token=self.token)

        fetched_metadata = [
            export.GetMetadataLegacy(client_fd, token=self.token)
            for client_fd in client_fds
        ]

      for metadata in fetched_metadata:
        metadata.source_urn = self.source_urn

        self._cached_metadata[metadata.client_urn] = metadata
        result[metadata.client_urn] = metadata
        metadata_to_fetch.remove(metadata.client_urn)

      for urn in metadata_to_fetch:
        default_mdata = export.ExportedMetadata(source_urn=self.source_urn)
        result[urn] = default_mdata
        self._cached_metadata[urn] = default_mdata

    return [result[urn] for urn in client_urns]
1191057
def get_dataset(self, datasetid):
        """"""

        path = '/api/1.0/meta/dataset/{}'
        return self._api_get(definition.Dataset, path.format(datasetid))
1131856
def _merge_metadata_dictionaries(cls, dictionaries):
        
        # three levels of nested dictionaries!
        #   {source name: {variant: {attribute: value}}}
        combined_dictionary = {}
        for source_to_metadata_dict in dictionaries:
            for source_name, variant_to_metadata_dict in source_to_metadata_dict.items():
                combined_dictionary.setdefault(source_name, {})
                combined_source_dict = combined_dictionary[source_name]
                for variant, metadata_dict in variant_to_metadata_dict.items():
                    combined_source_dict.setdefault(variant, {})
                    combined_source_dict[variant].update(metadata_dict)
        return combined_dictionary
609527
public void preFetchHBaseMeta() {
    LOG.info("Pre-fetching meta data for all tables");
    final long start = System.currentTimeMillis();
    final ArrayList<Deferred<Object>> deferreds = new ArrayList<Deferred<Object>>();
    deferreds.add(client.prefetchMeta(table));
    deferreds.add(client.prefetchMeta(uidtable));

    // TODO(cl) - meta, tree, etc

    try {
      Deferred.group(deferreds).join();
      LOG.info("Fetched meta data for tables in " +
          (System.currentTimeMillis() - start) + "ms");
    } catch (InterruptedException e) {
      LOG.error("Interrupted", e);
      Thread.currentThread().interrupt();
      return;
    } catch (Exception e) {
      LOG.error("Failed to prefetch meta for our tables", e);
    }
  }
980149
def get_sources_from_dataset(p, dataset, mode):
  """"""

  import apache_beam as beam
  import csv
  from google.datalab.ml import CsvDataSet, BigQueryDataSet

  check_dataset(dataset, mode)
  if type(dataset) is CsvDataSet:
    source_list = []
    for ii, input_path in enumerate(dataset.files):
      source_list.append(p | 'Read from Csv %d (%s)' % (ii, mode) >>
                         beam.io.ReadFromText(input_path, strip_trailing_newlines=True))
    return (source_list |
            'Flatten Sources (%s)' % mode >>
            beam.Flatten() |
            'Create Dict from Csv (%s)' % mode >>
            beam.Map(lambda line: csv.DictReader([line], fieldnames=['image_url',
                                                                     'label']).next()))
  elif type(dataset) is BigQueryDataSet:
    bq_source = (beam.io.BigQuerySource(table=dataset.table) if dataset.table is not None else
                 beam.io.BigQuerySource(query=dataset.query))
    return p | 'Read source from BigQuery (%s)' % mode >> beam.io.Read(bq_source)
  else:
    raise ValueError('Invalid DataSet. Expect CsvDataSet or BigQueryDataSet')
890684
def metadata(dataset, node, entityids, extended=False, api_key=None):
    
    api_key = _get_api_key(api_key)

    url = '{}/metadata'.format(USGS_API)
    payload = {
        "jsonRequest": payloads.metadata(dataset, node, entityids, api_key=api_key)
    }
    r = requests.post(url, payload)
    response = r.json()

    _check_for_usgs_error(response)

    if extended:
        metadata_urls = map(_get_metadata_url, response['data'])
        results = _async_requests(metadata_urls)
        data = map(lambda idx: _get_extended(response['data'][idx], results[idx]), range(len(response['data'])))

    return response
171700
def get_resource_data(ref_key, ref_id, scenario_id, type_id=None, expunge_session=True, **kwargs):
    

    user_id = kwargs.get('user_id')

    resource_data_qry = db.DBSession.query(ResourceScenario).filter(
        ResourceScenario.dataset_id   == Dataset.id,
        ResourceAttr.id == ResourceScenario.resource_attr_id,
        ResourceScenario.scenario_id == scenario_id,
        ResourceAttr.ref_key == ref_key,
        or_(
            ResourceAttr.network_id==ref_id,
            ResourceAttr.node_id==ref_id,
            ResourceAttr.link_id==ref_id,
            ResourceAttr.group_id==ref_id
        )).distinct().\
            options(joinedload('resourceattr')).\
            options(joinedload_all('dataset.metadata')).\
            order_by(ResourceAttr.attr_is_var)

    if type_id is not None:
        attr_ids = []
        rs = db.DBSession.query(TypeAttr).filter(TypeAttr.type_id==type_id).all()
        for r in rs:
            attr_ids.append(r.attr_id)

        resource_data_qry = resource_data_qry.filter(ResourceAttr.attr_id.in_(attr_ids))

    resource_data = resource_data_qry.all()

    for rs in resource_data:

        #TODO: Design a mechanism to read the value of the dataset if it's stored externally

        if rs.dataset.hidden == 'Y':
           try:
                rs.dataset.check_read_permission(user_id)
           except:
               rs.dataset.value      = None

    if expunge_session == True:
        db.DBSession.expunge_all()

    return resource_data
327082
public static void populateMetadata(AbstractComputeInstanceMetadata instanceMetadata, Map<?, ?> metadata) {
        if (metadata != null) {
            Map<String, String> finalMetadata = new HashMap<>(metadata.size());
            for (Map.Entry<?, ?> entry : metadata.entrySet()) {
                Object key = entry.getKey();
                Object value = entry.getValue();
                if (value instanceof String) {
                    finalMetadata.put(key.toString(), value.toString());
                }
            }
            instanceMetadata.setMetadata(finalMetadata);
        }
    }
479282
public IResultSet getDataSourceObjects( String catalog, String schema, String object, String version ) throws OdaException
	{
	    throw new UnsupportedOperationException();
	}
767907
def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
                url=None, resume=True, verbose=1):
    
    warn(DEPRECATION_MSG)
    file_id, md5 = OSF_RESOURCES[dataset_name]
    if url is None:
        url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
    return fetch_file(dataset_name, url, data_dir, dataset_prefix=dataset_prefix,
                      filetype='tar', resume=resume, verbose=verbose, md5sum=md5)
480052
def _get_basis_metadata(name, data_dir):
    '''
    '''

    # Transform the name into an internal representation
    tr_name = misc.transform_basis_name(name)

    # Get the metadata for all basis sets
    metadata = get_metadata(data_dir)

    if not tr_name in metadata:
        raise KeyError("Basis set {} does not exist".format(name))

    return metadata[tr_name]
726092
@Override
  public Metadata createMetadata() {
    Metadata metadata = new Metadata();
    try {
      for (DataSet ds : iimFile.getDataSets()) {
        Object value = "";
        try {
          value = ds.getValue();
        } catch (Exception ex) {

        }
        DataSetInfo info = ds.getInfo();
        //System.out.println(info.toString() + " " + info.getName() + ": " + value);

        metadata.add(info.getName(), new Text(value.toString()));
      }
    } catch (Exception ex) {
      /*Nothing to be shown*/
    }

    return metadata;
  }
1208971
def get_catalog_metadata(catalog, exclude_meta_fields=None):
    """"""
    exclude_meta_fields = exclude_meta_fields or []
    catalog_dict_copy = catalog.copy()
    del catalog_dict_copy["dataset"]

    for excluded_meta_field in exclude_meta_fields:
        catalog_dict_copy.pop(excluded_meta_field, None)

    return catalog_dict_copy
1061655
def get_metadata(self, dataset_identifier, content_type="json"):
        '''
        
        '''
        resource = _format_old_api_request(dataid=dataset_identifier, content_type=content_type)
        return self._perform_request("get", resource)