Load artifacts from the data store.
497528
def _LoadArtifactsFromDatastore(self):
    """"""
    loaded_artifacts = []

    # TODO(hanuszczak): Why do we have to remove anything? If some artifact
    # tries to shadow system artifact shouldn't we just ignore them and perhaps
    # issue some warning instead? The datastore being loaded should be read-only
    # during upload.

    # A collection of artifacts that shadow system artifacts and need
    # to be deleted from the data store.
    to_delete = []

    for artifact_coll_urn in self._sources.GetDatastores():
      artifact_coll = ArtifactCollection(artifact_coll_urn)

      if data_store.RelationalDBEnabled():
        artifact_list = data_store.REL_DB.ReadAllArtifacts()
      else:
        artifact_list = list(artifact_coll)

      for artifact_value in artifact_list:
        try:
          self.RegisterArtifact(
              artifact_value,
              source="datastore:%s" % artifact_coll_urn,
              overwrite_if_exists=True)
          loaded_artifacts.append(artifact_value)
          logging.debug("Loaded artifact %s from %s", artifact_value.name,
                        artifact_coll_urn)
        except rdf_artifacts.ArtifactDefinitionError as e:
          # TODO(hanuszczak): String matching on exception message is rarely
          # a good idea. Instead this should be refectored to some exception
          # class and then handled separately.
          if "system artifact" in str(e):
            to_delete.append(artifact_value.name)
          else:
            raise

    if to_delete:
      DeleteArtifactsFromDatastore(to_delete, reload_artifacts=False)
      self._dirty = True

      # TODO(hanuszczak): This is connected to the previous TODO comment. Why
      # do we throw exception at this point? Why do we delete something and then
      # abort the whole upload procedure by throwing an exception?
      detail = "system artifacts were shadowed and had to be deleted"
      raise rdf_artifacts.ArtifactDefinitionError(to_delete, detail)

    # Once all artifacts are loaded we can validate.
    revalidate = True
    while revalidate:
      revalidate = False
      for artifact_obj in loaded_artifacts[:]:
        try:
          Validate(artifact_obj)
        except rdf_artifacts.ArtifactDefinitionError as e:
          logging.error("Artifact %s did not validate: %s", artifact_obj.name,
                        e)
          artifact_obj.error_message = utils.SmartStr(e)
          loaded_artifacts.remove(artifact_obj)
          revalidate = True
793367
def load_data(self, pdbid):
        """"""
        f = urlopen("http://projects.biotec.tu-dresden.de/plip-rest/pdb/%s?format=xml" % pdbid.lower())
        self.doc = etree.parse(f)
84102
protected function loadFromStore()
    {
        if (false === $this->isLoaded()) {
            // Loads collection from the database on iteration.
            $models = $this->store->loadCollection($this);
            $this->setModels($models);
            $this->loaded = true;
        }
    }
820519
def __load(self, path):
        """"""
        try:
            path = os.path.abspath(path)
            with open(path, 'rb') as df:
                # loaded_dataset = pickle.load(df)
                self.__data, self.__classes, self.__labels, \
                self.__dtype, self.__description, \
                self.__num_features, self.__feature_names = pickle.load(df)

            # ensure the loaded dataset is valid
            self.__validate(self.__data, self.__classes, self.__labels)

        except IOError as ioe:
            raise IOError('Unable to read the dataset from file: {}', format(ioe))
        except:
            raise
68044
def _load_data(self, data, from_db=False):
        
        self._data = data[:]
        self.setattrs(
            values=[],
            node_stack=[],
            node_dict={},
        )
        self._from_db = from_db
1237914
private function load()
    {
        $this->cached = [];
        $entities = $this->daoGeneric->getEntities($this->entityName);
        foreach ($entities as $one) {
            $id = $one[ETypeBase::A_ID];
            $code = $one[ETypeBase::A_CODE];
            $this->cached[$code] = $id;
        }
    }
1145162
def load(self, ds, verbose=False):
        
        try:
            self.fetchone('select ID from dataset')
        except sqlite3.OperationalError:
            self.create(force=True)
        self.unload(ds)
        dataset = ds.cldf.wl
        tables, ref_tables = schema(dataset)

        # update the DB schema:
        for t in tables:
            if self._create_table_if_not_exists(t):
                continue
            db_cols = self.tables[t.name]
            for col in t.columns:
                if col.name not in db_cols:
                    with self.connection() as conn:
                        conn.execute(
                            "ALTER TABLE {0} ADD COLUMN `{1.name}` {1.db_type}".format(
                                t.name, col))
                else:
                    if db_cols[col.name] != col.db_type:
                        raise ValueError(
                            'column {0}:{1} {2} redefined with new type {3}'.format(
                                t.name, col.name, db_cols[col.name], col.db_type))

        for t in ref_tables.values():
            self._create_table_if_not_exists(t)

        self.update_schema()

        # then load the data:
        with self.connection() as db:
            db.execute('PRAGMA foreign_keys = ON;')
            insert(
                db,
                'dataset',
                'ID,name,version,metadata_json',
                (
                    ds.id,
                    '{0}'.format(dataset),
                    git_hash(ds.dir),
                    dumps(dataset.metadata_dict)))
            insert(
                db,
                'datasetmeta',
                'dataset_ID,key,value',
                *[(ds.id, k, '{0}'.format(v)) for k, v in dataset.properties.items()])

            # load sources:
            rows = []
            for src in dataset.sources.items():
                values = [ds.id, src.id, src.genre] + [src.get(k) for k in BIBTEX_FIELDS]
                values.append(
                    dumps({k: v for k, v in src.items() if k not in BIBTEX_FIELDS}))
                rows.append(tuple(values))
            insert(
                db,
                'SourceTable',
                ['dataset_ID', 'ID', 'bibtex_type'] + BIBTEX_FIELDS + ['extra'],
                *rows)

            # For regular tables, we extract and keep references to sources.
            refs = defaultdict(list)

            for t in tables:
                # We want to lookup columns by the name used in the CLDF dataset.
                cols = {col.cldf_name: col for col in t.columns}
                # But we also want to look up primary keys by the database column name.
                cols_by_name = {col.name: col for col in t.columns}

                ref_table = ref_tables.get(t.name)
                rows, keys = [], []
                try:
                    for row in dataset[t.name]:
                        keys, values = ['dataset_ID'], [ds.id]
                        for k, v in row.items():
                            if ref_table and k == ref_table.consumes:
                                col = cols_by_name[t.primary_key]
                                refs[ref_table.name].append((row[col.cldf_name], v))
                            else:
                                col = cols[k]
                                if isinstance(v, list):
                                    v = (col.separator or ';').join(
                                        nfilter(col.convert(vv) for vv in v))
                                else:
                                    v = col.convert(v)
                                keys.append("`{0}`".format(col.name))
                                values.append(v)
                        keys, values = self.update_row(t.name, keys, values)
                        rows.append(tuple(values))
                    insert(db, t.name, keys, *rows, **{'verbose': verbose})
                except FileNotFoundError:
                    if t.name != 'CognateTable':  # An empty CognateTable is allowed.
                        raise  # pragma: no cover

            # Now insert the references, i.e. the associations with sources:
            for tname, items in refs.items():
                rows = []
                for oid, sources in items:
                    for source in sources:
                        sid, context = Sources.parse(source)
                        rows.append([ds.id, oid, sid, context])
                oid_col = '{0}_ID'.format(tname.replace('Source', ''))
                insert(db, tname, ['dataset_ID', oid_col, 'Source_ID', 'Context'], *rows)
            db.commit()
1346120
def MigrateArtifacts():
  """"""

  # First, delete all existing artifacts in REL_DB.
  artifacts = data_store.REL_DB.ReadAllArtifacts()
  if artifacts:
    logging.info("Deleting %d artifacts from REL_DB.", len(artifacts))
    for artifact in data_store.REL_DB.ReadAllArtifacts():
      data_store.REL_DB.DeleteArtifact(Text(artifact.name))
  else:
    logging.info("No artifacts found in REL_DB.")

  artifacts = artifact_registry.REGISTRY.GetArtifacts(
      reload_datastore_artifacts=True)

  logging.info("Found %d artifacts in AFF4.", len(artifacts))

  # Only migrate user-created artifacts.
  artifacts = list(filter(_IsCustom, artifacts))

  logging.info("Migrating %d user-created artifacts.", len(artifacts))

  for artifact in artifacts:
    _MigrateArtifact(artifact)
1138028
def _save_artifact(build, data, content_type):
    """"""
    sha1sum = hashlib.sha1(data).hexdigest()
    artifact = models.Artifact.query.filter_by(id=sha1sum).first()

    if artifact:
      logging.debug('Upload already exists: artifact_id=%r', sha1sum)
    else:
      logging.info('Upload received: artifact_id=%r, content_type=%r',
                   sha1sum, content_type)
      artifact = models.Artifact(
          id=sha1sum,
          content_type=content_type,
          data=data)
      _artifact_created(artifact)

    artifact.owners.append(build)
    return artifact
166337
def _ReloadArtifacts(self):
    """"""
    self._artifacts = {}
    self._LoadArtifactsFromFiles(self._sources.GetAllFiles())
    self.ReloadDatastoreArtifacts()
97271
private function loadResources()
    {
        if ($this->class !== null && $this->classFilterSet === false) {
            $this->addClassFilter();
        }

        /** @var common_persistence_SqlPersistence $persistence */
        $persistence = $this->getModel()->getPersistence();
        $platform = $persistence->getPlatForm();
        $query = $platform->getQueryBuilder()
            ->select('*')
            ->from('statements')
            ->andWhere('id > ' . $this->lastId)
            ->orderBy('id');

        if ($this->cacheSize > 0) {
            $query->setMaxResults($this->cacheSize);
        }

        $query = $this->filter->applyFilters($query);
        $results = $query->execute();

        if ($this->cacheSize > 0) {
            $this->endOfClass = $results->rowCount() < $this->cacheSize;
        }

        foreach ($results->fetchAll() as $result) {
            $this->resources[] = $result;
            $this->lastId = $result['id'] > $this->lastId ? $result['id'] : $this->lastId;
        }
    }
768784
private function loadResourceMetadata()
    {
        foreach (get_declared_classes() as $className) {
            if (
                !isset($this->resources[$className]) &&
                is_subclass_of($className, 'Tonic\Resource')
            ) {
                $rm = new ResourceMetadata($className);
                if ($rm->getUri()) {
                    $this->resources[$className] = $rm;
                }
            }
        }
    }
598715
private function loadCacheMetaData()
    {
        if (isset($this->metaData)) {
            return;
        }

        $metaFile = $this->cacheDirectory . '/cache.meta';
        $this->metaData = array();

        if (file_exists($metaFile)) {
            $this->metaData = unserialize(file_get_contents($metaFile));
        }
    }
223239
def load(self):
        """"""
        if self.r.live:
            self.extra_config['collections'] = {'live':
                    {'index': '$live'}}

        if self.r.debug:
            self.extra_config['debug'] = True

        if self.r.record:
            self.extra_config['recorder'] = 'live'
655850
def _load_resources(self):
        """"""

        for t in self.doc.find('Root.Table'):
            for c in t.find('Table.Column'):
                if c.get_value('datatype') == 'geometry':
                    c['transform'] = '^empty_str'
                    c['datatype'] = 'text'

        return super()._load_resources()
635746
def ReadArtifact(self, name):
    """"""
    try:
      artifact = self.artifacts[name]
    except KeyError:
      raise db.UnknownArtifactError(name)

    return artifact.Copy()
397965
def _loadData(self, data):
        
        self._data = data
        self.id = utils.cast(int, data.attrib.get('id'))
        self.serverId = utils.cast(int, data.attrib.get('serverId'))
        self.machineIdentifier = data.attrib.get('machineIdentifier')
        self.name = data.attrib.get('name')
        self.lastSeenAt = utils.toDatetime(data.attrib.get('lastSeenAt'))
        self.numLibraries = utils.cast(int, data.attrib.get('numLibraries'))
        self.allLibraries = utils.cast(bool, data.attrib.get('allLibraries'))
        self.owned = utils.cast(bool, data.attrib.get('owned'))
        self.pending = utils.cast(bool, data.attrib.get('pending'))