Generate local DB, pulling metadata and data from RWSConnection
327262
def execute(self):
        """"""

        logging.info('Requesting view metadata for project %s' % self.project_name)
        project_csv_meta = self.rws_connection.send_request(ProjectMetaDataRequest(self.project_name))

        # Process it into a set of tables
        self.db_adapter.processMetaData(project_csv_meta)

        # Get the data for the study
        for dataset_name in self.db_adapter.datasets.keys():
            logging.info('Requesting data from dataset %s' % dataset_name)
            form_name, _type = self.name_type_from_viewname(dataset_name)
            form_data = self.rws_connection.send_request(
                FormDataRequest(self.project_name, self.environment, _type, form_name))

            # Now process the form_data into the db of choice
            logging.info('Populating dataset %s' % dataset_name)
            self.db_adapter.processFormData(form_data, dataset_name)

        logging.info('Process complete')
476123
function readData(datapath, layer, callback) {
  var features = [];

  var options = {
    delimiter: ',',
    columns: true
  };

  fs.createReadStream(path.join(datapath, 'meta', `wof-${layer}-latest.csv`))
    .pipe(parse(options))
    .pipe(loadJSON.create(datapath))
    .pipe(isActiveRecord.create())
    .pipe(filterOutNamelessRecords.create())
    .pipe(filterOutUnimportantRecords.create())
    .pipe(extractFields.create())
    .pipe(simplifyGeometry.create())
    .pipe(sink.obj(function(feature) {
      features.push(feature);
    }))
    .on('finish', function() {
      callback(features);
    });

}
453247
def _connect(dbfile: 'PathLike') -> apsw.Connection:
    """"""
    conn = apsw.Connection(os.fspath(dbfile))
    _set_foreign_keys(conn, 1)
    assert _get_foreign_keys(conn) == 1
    return conn
943742
def populate_db(sql_path):
    """"""

    logger.info("Populating DB %s from %s", repr(DB["NAME"]), repr(sql_path))
    shell(
        'psql -U "{USER}" -h "{HOST}" -d "{NAME}" --file={sql_path}'.format(
            sql_path=sql_path, **DB
        )
    )
38175
def external_metadata(self, datasource_type=None, datasource_id=None):
        """"""
        if datasource_type == 'druid':
            datasource = ConnectorRegistry.get_datasource(
                datasource_type, datasource_id, db.session)
        elif datasource_type == 'table':
            database = (
                db.session
                .query(Database)
                .filter_by(id=request.args.get('db_id'))
                .one()
            )
            Table = ConnectorRegistry.sources['table']
            datasource = Table(
                database=database,
                table_name=request.args.get('table_name'),
                schema=request.args.get('schema') or None,
            )
        external_metadata = datasource.external_metadata()
        return self.json_response(external_metadata)
725637
def generate_fixtures(datasets, reuses):
    ''''''
    user = UserFactory()
    log.info('Generated user "{user.email}".'.format(user=user))

    organization = OrganizationFactory(members=[Member(user=user)])
    log.info('Generated organization "{org.name}".'.format(org=organization))

    for _ in range(datasets):
        dataset = VisibleDatasetFactory(organization=organization)
        DiscussionFactory(subject=dataset, user=user)
        ReuseFactory.create_batch(reuses, datasets=[dataset], owner=user)

    msg = 'Generated {datasets} dataset(s) with {reuses} reuse(s) each.'
    log.info(msg.format(**locals()))
132450
def connect_db(Repo, database=":memory:"):
        
        Repo.db = sqlite3.connect(database,
                                  detect_types=sqlite3.PARSE_DECLTYPES)
        return Repo.db
171700
def get_resource_data(ref_key, ref_id, scenario_id, type_id=None, expunge_session=True, **kwargs):
    

    user_id = kwargs.get('user_id')

    resource_data_qry = db.DBSession.query(ResourceScenario).filter(
        ResourceScenario.dataset_id   == Dataset.id,
        ResourceAttr.id == ResourceScenario.resource_attr_id,
        ResourceScenario.scenario_id == scenario_id,
        ResourceAttr.ref_key == ref_key,
        or_(
            ResourceAttr.network_id==ref_id,
            ResourceAttr.node_id==ref_id,
            ResourceAttr.link_id==ref_id,
            ResourceAttr.group_id==ref_id
        )).distinct().\
            options(joinedload('resourceattr')).\
            options(joinedload_all('dataset.metadata')).\
            order_by(ResourceAttr.attr_is_var)

    if type_id is not None:
        attr_ids = []
        rs = db.DBSession.query(TypeAttr).filter(TypeAttr.type_id==type_id).all()
        for r in rs:
            attr_ids.append(r.attr_id)

        resource_data_qry = resource_data_qry.filter(ResourceAttr.attr_id.in_(attr_ids))

    resource_data = resource_data_qry.all()

    for rs in resource_data:

        #TODO: Design a mechanism to read the value of the dataset if it's stored externally

        if rs.dataset.hidden == 'Y':
           try:
                rs.dataset.check_read_permission(user_id)
           except:
               rs.dataset.value      = None

    if expunge_session == True:
        db.DBSession.expunge_all()

    return resource_data
1084813
def generatemetadata(self, parameters, parentfile, relevantinputfiles, provenancedata = None):
        """"""
        assert isinstance(provenancedata,CLAMProvenanceData) or provenancedata is None

        data = {}

        if self.copymetadata:
            #Copy parent metadata
            for key, value in parentfile.metadata.items():
                data[key] = value

        for metafield in self.metafields:
            if isinstance(metafield, ParameterCondition):
                metafield = metafield.evaluate(parameters)
                if not metafield:
                    continue
            assert isinstance(metafield, AbstractMetaField)
            metafield.resolve(data, parameters, parentfile, relevantinputfiles)

        if provenancedata:
            data['provenance'] = provenancedata

        return self.formatclass(None, **data)
69103
def create_metadata(name,
                    ext_version=None,
                    schema=None,
                    user=None,
                    host=None,
                    port=None,
                    maintenance_db=None,
                    password=None,
                    runas=None):
    '''
    

    '''
    installed_ext = get_installed_extension(
        name,
        user=user,
        host=host,
        port=port,
        maintenance_db=maintenance_db,
        password=password,
        runas=runas)
    ret = [_EXTENSION_NOT_INSTALLED]
    if installed_ext:
        ret = [_EXTENSION_INSTALLED]
        if (
            ext_version is not None
            and _pg_is_older_ext_ver(
                installed_ext.get('extversion', ext_version),
                ext_version
            )
        ):
            ret.append(_EXTENSION_TO_UPGRADE)
        if (
            schema is not None
            and installed_ext.get('extrelocatable', 'f') == 't'
            and installed_ext.get('schema_name', schema) != schema
        ):
            ret.append(_EXTENSION_TO_MOVE)
    return ret
164662
def export_datasources(print_stdout, datasource_file,
                       back_references, include_defaults):
    """"""
    data = dict_import_export.export_to_dict(
        session=db.session,
        recursive=True,
        back_references=back_references,
        include_defaults=include_defaults)
    if print_stdout or not datasource_file:
        yaml.safe_dump(data, stdout, default_flow_style=False)
    if datasource_file:
        logging.info('Exporting datasources to %s', datasource_file)
        with open(datasource_file, 'w') as data_stream:
            yaml.safe_dump(data, data_stream, default_flow_style=False)
492756
def _datatable_factory(self):
        
        feed_cols = ['feed{0:03d}'.format(i + 1) for i in range(self.n_feeds)]
        feed_cols = ['override_feed000'] + feed_cols + ['failsafe_feed999']

        ind_sqlatyp = indexingtypes[self.index.indimp].sqlatyp
        dat_sqlatyp = datadefs[self.dtype.datadef].sqlatyp

        atbl = Table(self.name, Base.metadata,
                     Column('indx', ind_sqlatyp, primary_key=True),
                     Column('final', dat_sqlatyp),
                     *(Column(fed_col, dat_sqlatyp) for fed_col in feed_cols),
                     extend_existing=True)
        
        self.dt_feed_cols = feed_cols[:]
        self.dt_all_cols = ['indx', 'final'] + feed_cols[:]
        return atbl
1012415
public void gatherAndDisplayMetaDataInfo(Connection conn, WSManagedConnectionFactoryImpl mcf) throws SQLException
    {
        java.sql.DatabaseMetaData mData = conn.getMetaData();

        String databaseProductName = mData.getDatabaseProductName();
        String driverName = mData.getDriverName(); 

        String driverVersion = null;
        String databaseProductVersion = null;
        try {
            driverVersion = mData.getDriverVersion();
            databaseProductVersion = mData.getDatabaseProductVersion();
        } catch (Exception e) {
            // Ignore any Runtime errors as the info collected are solely for FFDC purpose.
            if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                Tr.debug(this, tc, "Exception occurred while getting metaData info. Exception is: ", e);
            }
        }

        setDatabaseProductName(databaseProductName);
        setDriverName(driverName); 
        setDriverMajorVersion(mData.getDriverMajorVersion()); 

        Tr.info(tc, "DB_PRODUCT_NAME", databaseProductName);
        Tr.info(tc, "DB_PRODUCT_VERSION", databaseProductVersion); 
        Tr.info(tc, "JDBC_DRIVER_NAME", driverName); 
        Tr.info(tc, "JDBC_DRIVER_VERSION", driverVersion); 

        try {
            if (mData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT) &&
                mData.supportsResultSetHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT)) {
                if (tc.isDebugEnabled()) {
                    Tr.debug(this, tc, "Holdabiliy is supported");
                }
                holdabilitySupported = true;
            } else {
                if (tc.isDebugEnabled()) {
                    Tr.debug(this, tc, "Holdability not supported");
                }
                holdabilitySupported = false;
            }

        } catch (Throwable x) {
            if (x instanceof SQLException) {
                SQLException sqe = (SQLException) x;
                if ((isConnectionError(sqe))) {
                    throw sqe; // if this is a stale we need to throw exception here.
                }

                // Some DB drivers throw SQLException when the JDBC version is 2.x.
                // Holdability is not supported in 2.0

                holdabilitySupported = false;
                if (tc.isDebugEnabled()) {
                    Tr.debug(this, tc, "getHoldability is not supported in this JDBC driver. Encounter a java.sql.SQLException", sqe);
                }
            } else 
            {
                //NO FFDC needed
                // just log it and do nothing
                if (tc.isDebugEnabled()) {
                    Tr.debug(this, tc, "caught an exception when testing the holdability from metadata, will call the holdability itself to know if supported", x);
                }

            }
        }
        try {
            mcf.jdbcDriverSpecVersion = mData.getJDBCMajorVersion() * 10;
        } catch (AbstractMethodError methError) {
            // No FFDC code needed. JDBC driver version is less than 3.0.
            mcf.jdbcDriverSpecVersion = 20;
        }
        catch (UnsupportedOperationException supportX) // raised by old Sybase driver
        {
            // No FFDC code needed. Driver is not JDBC 3.0 compliant because it does not
            // support getJDBCMajorVersion,
            mcf.jdbcDriverSpecVersion = 20;
        }
        catch (SQLException sqlX) {
            // No FFDC code needed.
            // Informix driver throws SQLException when the JDBC version is 2.x.

            mcf.jdbcDriverSpecVersion = 20;

            if (tc.isDebugEnabled())
                Tr.debug(this, tc, "metadata.getJDBCMajorVersion", sqlX);

            // Need to fail if it was a connection error.
            if (isConnectionError(sqlX))
                throw sqlX;
        }

        if (mcf.jdbcDriverSpecVersion > 30)
            try {
                mcf.jdbcDriverSpecVersion += mData.getJDBCMinorVersion();
            } catch (SQLException x) {
            } catch (UnsupportedOperationException x) {
            }

        if (mcf.jdbcDriverSpecVersion >= 40)
            try 
            {
                // Reset to the default client info of the data source (based on an initial connection),
                // instead of using the default values for the JDBC driver--which could differ from the
                // defaults specified for the data source.
                Properties clientInfo = conn.getClientInfo();
                if (clientInfo != null)
                    mcf.defaultClientInfo.putAll(clientInfo);
            }
            catch (SQLFeatureNotSupportedException featureX) {
                // No FFDC code needed.
                // The JDBC 4.0 driver ought to implement the getClientInfOProperties method,
                // but doesn't.  Issue a warning, but tolerate the error so that WAS can still
                // interoperate with the JDBC driver.
                Tr.warning(tc, "FEATURE_NOT_IMPLEMENTED", "java.sql.Connection.getClientInfo");
            }
            catch (AbstractMethodError methErr) {
                // No FFDC code needed.
                // Work around a bug in Oracle where the driver reports "10" instead
                // of the JDBC specification level.
                // If the JDBC 4.0 metadata operation cannot be found on the driver,
                // then it must really be a JDBC 3.0 driver.
                if (tc.isDebugEnabled())
                    Tr.debug(this, tc,
                             "JDBC spec level, " + mcf.jdbcDriverSpecVersion +
                                             ", reported by the driver is not valid. Using 30 (3.0) instead.");
                mcf.jdbcDriverSpecVersion = 30;
            }
            catch (UnsupportedOperationException operationX) 
            {
                // No FFDC code needed.
                // Returned by Sybase jconn4.jar versions that are older than jConnect-7.07.
                // The JDBC 4.0 driver ought to throw SQLFeatureNotSupportedException,
                // but doesn't.  Issue a warning, but tolerate the error so that WAS can still
                // interoperate with the JDBC driver.
                Tr.warning(tc, "FEATURE_NOT_IMPLEMENTED", "java.sql.Connection.getClientInfo");
            }

        if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled())
            Tr.debug(this, tc, "JDBC spec version implemented by driver", mcf.jdbcDriverSpecVersion);
    }
838462
def sync_db():
    
    with cd('/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])):
        venv = '/'.join([deployment_root(),'env',env.project_fullname,'bin','activate'])
        sites = _get_django_sites()
        site_ids = sites.keys()
        site_ids.sort()
        for site in site_ids:
            for settings_file in _sitesettings_files():
                site_settings = '.'.join([env.project_package_name,'sitesettings',settings_file.replace('.py','')])
                if env.verbosity:
                    print " * django-admin.py syncdb --noinput --settings=%s"% site_settings
                output = sudo(' '.join(['source',venv,'&&',"django-admin.py syncdb --noinput --settings=%s"% site_settings]),
                              user='site_%s'% site)
                if env.verbosity:
                    print output
288899
def _get_metadata(network_id, user_id):
    
    log.info("Getting Metadata")
    dataset_qry = db.DBSession.query(
                Dataset
    ).outerjoin(DatasetOwner, and_(DatasetOwner.dataset_id==Dataset.id, DatasetOwner.user_id==user_id)).filter(
                or_(Dataset.hidden=='N', DatasetOwner.user_id != None),
                Scenario.id==ResourceScenario.scenario_id,
                Scenario.network_id==network_id,
                Dataset.id==ResourceScenario.dataset_id).distinct().subquery()

    rs_qry = db.DBSession.query(
                Metadata
    ).join(dataset_qry, Metadata.dataset_id==dataset_qry.c.id)

    x = time.time()
    logging.info("Getting all matadata")
    all_metadata = db.DBSession.execute(rs_qry.statement).fetchall()
    log.info("%s metadata jointly retrieved in %s",len(all_metadata), time.time()-x)

    logging.info("metadata retrieved. Processing results...")
    x = time.time()
    metadata_dict = dict()
    for m in all_metadata:
        if metadata_dict.get(m.dataset_id):
            metadata_dict[m.dataset_id][m.key] = six.text_type(m.value)
        else:
            metadata_dict[m.dataset_id] = {m.key : six.text_type(m.value)}

    logging.info("metadata processed in %s", time.time()-x)

    return metadata_dict
903848
def syncdb():
    
    from flask_philo.db.postgresql.schema import Base
    from flask_philo.db.postgresql.orm import BaseModel  # noqa
    from flask_philo.db.postgresql.connection import get_pool

    for conn_name, conn in get_pool().connections.items():
        Base.metadata.create_all(conn.engine)
183754
def sync_remote_to_local(force="no"):
    

    assert "local_wp_dir" in env, "Missing local_wp_dir in env"

    if force != "yes":
        message = "This will replace your local database with your "\
            "remote, are you sure [y/n]"
        answer = prompt(message, "y")

        if answer != "y":
            logger.info("Sync stopped")
            return

    init_tasks()  # Bootstrap fabrik

    remote_file = "sync_%s.sql" % int(time.time()*1000)
    remote_path = "/tmp/%s" % remote_file

    with env.cd(paths.get_current_path()):
        env.run("wp db export %s" % remote_path)

    local_wp_dir = env.local_wp_dir
    local_path = "/tmp/%s" % remote_file

    # Download sync file
    get(remote_path, local_path)

    with lcd(local_wp_dir):
        elocal("wp db import %s" % local_path)

    # Cleanup
    env.run("rm %s" % remote_path)
    elocal("rm %s" % local_path)