Be aware of data set lifecycle events
199757
void onDataSetDefRegisteredEvent(@Observes DataSetDefRegisteredEvent event) {
        PortablePreconditions.checkNotNull("DataSetDefRegisteredEvent",
                                           event);
        view.setDataSetCount(++dataSetCount);
    }
1271924
def purge(datasets, reuses, organizations):
    '''
    
    '''
    purge_all = not any((datasets, reuses, organizations))

    if purge_all or datasets:
        log.info('Purging datasets')
        purge_datasets()

    if purge_all or reuses:
        log.info('Purging reuses')
        purge_reuses()

    if purge_all or organizations:
        log.info('Purging organizations')
        purge_organizations()

    success('Done')
1097609
function(dataset){
    arrayRemove(this.sources, dataset);

    // remove event listeners from dataset
    if (this.listen.source)
      dataset.removeHandler(this.listen.source, this);

    // process removing dataset objects and update member map
    var memberMap = this.members_;
    for (var objectId in dataset.items_)
      memberMap[objectId].count--;
  }
1113087
private void createJobPropsForDatasets() {
    final Set<Dataset> datasetsWithProps = Sets.newHashSet();
    for (Dataset dataset : this.datasets) {
      datasetsWithProps.addAll(createJobPropsForDataset(dataset));
    }

    this.datasets.clear();
    this.datasets.addAll(datasetsWithProps);
  }
701503
static void dataSetAssertion(CallInfo callInfo, DataSet expected, DataSet actual) {
    validateDataSetAssertion(expected, actual);
    DataSource source = expected.getSource();
    Delta delta = new Delta(expected, actual); 
    DataSetAssertion assertion = new DataSetAssertion(expected, delta);
    source.getDB().log(callInfo, assertion);
    
    if (! assertion.passed()) {
      throw new DBAssertionError(callInfo.getMessage());
    }
  }
192285
def addDataset(self, dataset):
        
        self._datasets.append(dataset)
        self._dataChanged = True
        self._addDatasetAction(dataset)
625428
def check_dataset(dataset, mode):
  """"""

  names = [x['name'] for x in dataset.schema]
  types = [x['type'] for x in dataset.schema]
  if mode == 'train':
    if (set(['image_url', 'label']) != set(names) or any(t != 'STRING' for t in types)):
      raise ValueError('Invalid dataset. Expect only "image_url,label" STRING columns.')
  else:
    if (set(['image_url']) != set(names) and set(['image_url', 'label']) != set(names)) or \
            any(t != 'STRING' for t in types):
      raise ValueError('Invalid dataset. Expect only "image_url" or "image_url,label" ' +
                       'STRING columns.')
571201
private void onDataObjectRename(@Observes DataObjectRenamedEvent event) {
        Path updatedPath = event.getPath();
        if (updatedPath != null) {
            try {
                updateDataObject(updatedPath);
            } catch (Exception e) {
                logger.error("Data object couldn't be updated, path: " + updatedPath + ".");
            }
        }
    }
254311
function Dataset(sc, dependencies) {
  this.id = sc.datasetIdCounter++;
  this.dependencies = dependencies || [];
  this.persistent = false;
  this.sc = sc;
}
419345
private void createDataset(String datasetId, @Nullable String location)
      throws IOException, InterruptedException {
    Dataset dataset = new Dataset();
    DatasetReference reference = new DatasetReference();
    reference.setProjectId(projectId);
    reference.setDatasetId(datasetId);
    dataset.setDatasetReference(reference);
    if (location != null) {
      dataset.setLocation(location);
    }

    executeWithBackOff(
        client.datasets().insert(projectId, dataset),
        String.format(
            "Error when trying to create the temporary dataset %s in project %s.",
            datasetId, projectId));
  }
82788
function() {
		var me = this;
		helpers$1.each(me.data.datasets, function(dataset, datasetIndex) {
			me.getDatasetMeta(datasetIndex).controller.reset();
		}, me);
	}
1229357
def ():
    """"""
    clean_dir = ev(
        "OUTPUT_DIR",
        "/tmp")
    clean_file = ev(
        "CLEANED_FILE",
        "{}/cleaned_attack_scans.csv".format(
            clean_dir))
    fulldata_file = ev(
        "FULLDATA_FILE",
        "{}/fulldata_attack_scans.csv".format(
            clean_dir))
    dataset_dir = ev(
        "DS_DIR",
        "/opt/antinex/datasets")
    csv_glob_path = ev(
        "DS_GLOB_PATH",
        "{}/*/*.csv".format(
            dataset_dir))

    pipeline_files = find_all_pipeline_csvs(
        csv_glob_path=csv_glob_path)

    post_proc_rules = {
        "drop_columns": [
            "src_file",
            "raw_id",
            "raw_load",
            "raw_hex_load",
            "raw_hex_field_load",
            "pad_load",
            "eth_dst",  # need to make this an int
            "eth_src",  # need to make this an int
            "ip_dst",   # need to make this an int
            "ip_src"    # need to make this an int
        ],
        "predict_feature": "label_name"
    }

    label_rules = {
        "set_if_above": 85,
        "labels": ["not_attack", "attack"],
        "label_values": [0, 1]
    }

    log.info("building csv")

    save_node = build_csv(
        pipeline_files=pipeline_files,
        fulldata_file=fulldata_file,
        clean_file=clean_file,
        post_proc_rules=post_proc_rules,
        label_rules=label_rules)

    if save_node["status"] == VALID:
        log.info("Successfully process datasets:")

        if ev("SHOW_SUMMARY", "1") == "1":
            log.info(("Full csv: {}")
                     .format(save_node["fulldata_file"]))
            log.info(("Full meta: {}")
                     .format(save_node["fulldata_metadata_file"]))
            log.info(("Clean csv: {}")
                     .format(save_node["clean_file"]))
            log.info(("Clean meta: {}")
                     .format(save_node["clean_metadata_file"]))
            log.info("------------------------------------------")
            log.info(("Predicting Feature: {}")
                     .format(save_node["feature_to_predict"]))
            log.info(("Features to Process: {}")
                     .format(ppj(save_node["features_to_process"])))
            log.info(("Ignored Features: {}")
                     .format(ppj(save_node["ignore_features"])))
            log.info("------------------------------------------")
        # end of show summary

        log.info("")
        log.info("done saving csv:")
        log.info("Full: {}".format(
            save_node["fulldata_file"]))
        log.info("Cleaned (no-NaNs in columns): {}".format(
            save_node["clean_file"]))
        log.info("")
    else:
        log.info("Failed to process datasets")
766683
public void invalidateDataSet () {
		updateCurrentItem(MathUtils.clamp(currentIndex, 0, items.size - 1));
		spinner.notifyValueChanged(true);
	}
222088
def process(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    data(dataset, **ctx.obj).process(**kwargs)
1012942
def _cache_dataset(dataset, prefix):
    
    if not os.path.exists(_constants.CACHE_PATH):
        os.makedirs(_constants.CACHE_PATH)
    src_data = np.concatenate([e[0] for e in dataset])
    tgt_data = np.concatenate([e[1] for e in dataset])
    src_cumlen = np.cumsum([0]+[len(e[0]) for e in dataset])
    tgt_cumlen = np.cumsum([0]+[len(e[1]) for e in dataset])
    np.savez(os.path.join(_constants.CACHE_PATH, prefix + '.npz'),
             src_data=src_data, tgt_data=tgt_data,
             src_cumlen=src_cumlen, tgt_cumlen=tgt_cumlen)
122603
def clear_cache(dataset_name=None):
    """"""
    dr = data_resources[dataset_name]
    if 'dirs' in dr:
        for dirs, files in zip(dr['dirs'], dr['files']):
            for dir, file in zip(dirs, files):
                path = os.path.join(data_path, dataset_name, dir, file)
                if os.path.exists(path):
                    logging.info("clear_cache: removing " + path)
                    os.unlink(path)
            for dir in dirs:
                path = os.path.join(data_path, dataset_name, dir)
                if os.path.exists(path):
                    logging.info("clear_cache: remove directory " + path)
                    os.rmdir(path)
        
    else:
        for file_list in dr['files']:
            for file in file_list:
                path = os.path.join(data_path, dataset_name, file)
                if os.path.exists(path):
                    logging.info("clear_cache: remove " + path)
                    os.unlink(path)
215673
def _network_event_lifecycle_cb(conn, net, event, detail, opaque):
    '''
    
    '''

    _salt_send_event(opaque, conn, {
        'network': {
            'name': net.name(),
            'uuid': net.UUIDString()
        },
        'event': _get_libvirt_enum_string('VIR_NETWORK_EVENT_', event),
        'detail': 'unknown'  # currently unused
    })