Classifies the provided dataset samples.

@param string $uniqueid
@param \stored_file $dataset
@param string $outputdir
@return \stdClass
967710
public function classify($uniqueid, \stored_file $dataset, $outputdir) {

        // Obtain the physical route to the file.
        $datasetpath = $this->get_file_path($dataset);

        $cmd = "{$this->pathtopython} -m moodlemlbackend.prediction " .
            escapeshellarg($uniqueid) . ' ' .
            escapeshellarg($outputdir) . ' ' .
            escapeshellarg($datasetpath);

        if (!PHPUNIT_TEST && CLI_SCRIPT) {
            debugging($cmd, DEBUG_DEVELOPER);
        }

        $output = null;
        $exitcode = null;
        $result = exec($cmd, $output, $exitcode);

        if (!$result) {
            throw new \moodle_exception('errornopredictresults', 'analytics');
        }

        if (!$resultobj = json_decode($result)) {
            throw new \moodle_exception('errorpredictwrongformat', 'analytics', '', json_last_error_msg());
        }

        if ($exitcode != 0) {
            if (!empty($resultobj->errors)) {
                $errors = $resultobj->errors;
                if (is_array($errors)) {
                    $errors = implode(', ', $errors);
                }
            } else if (!empty($resultobj->info)) {
                // Show info if no errors are returned.
                $errors = $resultobj->info;
                if (is_array($errors)) {
                    $errors = implode(', ', $errors);
                }
            }
            $resultobj->info = array(get_string('errorpredictionsprocessor', 'analytics', $errors));
        }

        return $resultobj;
    }
419345
private void createDataset(String datasetId, @Nullable String location)
      throws IOException, InterruptedException {
    Dataset dataset = new Dataset();
    DatasetReference reference = new DatasetReference();
    reference.setProjectId(projectId);
    reference.setDatasetId(datasetId);
    dataset.setDatasetReference(reference);
    if (location != null) {
      dataset.setLocation(location);
    }

    executeWithBackOff(
        client.datasets().insert(projectId, dataset),
        String.format(
            "Error when trying to create the temporary dataset %s in project %s.",
            datasetId, projectId));
  }
268163
public function getIterator(string $uniqueId, ?int $gridId = null): GridIterator
    {
        if (!isset($this->iterators[$uniqueId])) {
            $grid     = $this->getGrid($gridId);
            $iterator = new GridIterator($grid);

            $this->iterators[$uniqueId] = $iterator;
        }

        return $this->iterators[$uniqueId];
    }
329959
protected function findOrCreateCategory(DataSetInterface $dataSet)
    {
        $categoryEntity = SpyCategoryQuery::create()
            ->filterByCategoryKey($dataSet[static::KEY_CATEGORY_KEY])
            ->findOneOrCreate();

        $categoryEntity->fromArray($dataSet->getArrayCopy());

        if (!empty($dataSet[static::KEY_TEMPLATE_NAME])) {
            $categoryTemplateEntity = $this->getCategoryTemplate($dataSet);
            $categoryEntity->setFkCategoryTemplate($categoryTemplateEntity->getIdCategoryTemplate());
        }

        if ($categoryEntity->isNew() || $categoryEntity->isModified()) {
            $categoryEntity->save();
        }

        return $categoryEntity;
    }
1299570
protected function validateDataSet(DataSetInterface $dataSet): void
    {
        if (!$dataSet[MerchantDataSetInterface::MERCHANT_KEY]) {
            throw new InvalidDataException('"' . MerchantDataSetInterface::MERCHANT_KEY . '" is required.');
        }

        if (!$dataSet[MerchantDataSetInterface::NAME]) {
            throw new InvalidDataException('"' . MerchantDataSetInterface::NAME . '" is required.');
        }
    }
741987
public function create($dataset)
    {
        // Event
        $this->triggerEvent(
            'onBefore' . ucfirst(__FUNCTION__),
            [
                'dataset' => &$dataset,
            ]
        );

        if (!($dataset instanceof \Traversable) && !is_array($dataset)) {
            throw new \InvalidArgumentException('DataSet object should be instance of a Traversable');
        }

        $result = $this->doCreate($dataset);

        // Event
        $this->triggerEvent(
            'onAfter' . ucfirst(__FUNCTION__),
            [
                'result' => &$result,
            ]
        );

        return $result;
    }
1023594
public function execute(DataSetInterface $dataSet)
    {
        $mimeTypeEntity = SpyMimeTypeQuery::create()
            ->filterByName($dataSet[MimeTypeDataSetInterface::KEY_NAME])
            ->findOneOrCreate();

        $mimeTypeEntity->setIsAllowed($dataSet[MimeTypeDataSetInterface::KEY_IS_ALLOWED]);
        $mimeTypeEntity->save();
    }
141583
public function execute(DataSetInterface $dataSet): void
    {
        if (!$dataSet[MerchantRelationshipDataSetInterface::COMPANY_BUSINESS_UNIT_ASSIGNEE_KEYS]) {
            $dataSet[MerchantRelationshipDataSetInterface::ID_COMPANY_BUSINESS_UNIT_ASSIGNEE_COLLECTION] = [];

            return;
        }

        $companyBusinessUnitKeys = explode(
            $this->getAssigneeDelimiter(),
            $dataSet[MerchantRelationshipDataSetInterface::COMPANY_BUSINESS_UNIT_ASSIGNEE_KEYS]
        );

        $companyBusinessUnitAssignee = [];
        foreach ($companyBusinessUnitKeys as $companyBusinessUnitKey) {
            $companyBusinessUnitAssignee[] = $this->findIdCompanyBusinessUnit($companyBusinessUnitKey);
        }

        $dataSet[MerchantRelationshipDataSetInterface::ID_COMPANY_BUSINESS_UNIT_ASSIGNEE_COLLECTION] = $companyBusinessUnitAssignee;
    }
940131
protected function checkIfProductHasConcreteOrAbstractAlternative(DataSetInterface $dataSet): void
    {
        if (!$dataSet[ProductAlternativeDataSetInterface::KEY_COLUMN_ALTERNATIVE_PRODUCT_CONCRETE_SKU] &&
            !$dataSet[ProductAlternativeDataSetInterface::KEY_COLUMN_ALTERNATIVE_PRODUCT_ABSTRACT_SKU]
        ) {
            throw new InvalidDataException(
                sprintf(
                    'Product concrete with SKU "%s" has neither concrete nor abstract alternative products',
                    $dataSet[ProductAlternativeDataSetInterface::KEY_COLUMN_CONCRETE_SKU]
                )
            );
        }
    }
733329
public function update(Dataset $dataset) : void
    {
        if (!$dataset->homogeneous() or $dataset->columnType(0) !== DataType::CONTINUOUS) {
            throw new InvalidArgumentException('This transformer only works'
                . ' with continuous features.');
        }

        if ($this->dfs === null or $this->n === null) {
            $this->fit($dataset);
            
            return;
        }

        foreach ($dataset as $sample) {
            foreach ($sample as $column => $feature) {
                if ($feature > 0) {
                    $this->dfs[$column]++;
                }
            }
        }

        $this->n += $dataset->numRows();

        $idfs = [];

        foreach ($this->dfs as $df) {
            $idfs[] = log($this->n / $df) + 1.;
        }

        $this->idfs = $idfs;
    }
802855
public function rank(Dataset $dataset) : array
    {
        if ($this->tree->bare()) {
            throw new RuntimeException('The learner has not'
                . ' been trained.');
        }
        
        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        return array_map([self::class, 'localOutlierFactor'], $dataset->samples());
    }
1175785
@Override
    public DataSet sample(int numSamples, boolean withReplacement) {
        return sample(numSamples, Nd4j.getRandom(), withReplacement);
    }
292706
def new_dataset(args):
    
    if not args.args:
        raise ParserError('you must specify an existing directory')
    outdir = Path(args.args.pop(0))
    if not outdir.exists():
        raise ParserError('you must specify an existing directory')

    id_pattern = re.compile('[a-z_0-9]+$')
    md = {}
    if args.args:
        md['id'] = args.args.pop(0)
    else:
        md['id'] = input('Dataset ID: ')

    while not id_pattern.match(md['id']):
        print('dataset id must only consist of lowercase ascii letters, digits and _ (underscore)!')
        md['id'] = input('Dataset ID: ')

    outdir = outdir / md['id']
    if not outdir.exists():
        outdir.mkdir()

    for key in ['title', 'url', 'license', 'conceptlist', 'citation']:
        md[key] = input('Dataset {0}: '.format(key))

    # check license!
    # check conceptlist!

    for path in Path(pylexibank.__file__).parent.joinpath('dataset_template').iterdir():
        if path.is_file():
            if path.suffix in ['.pyc']:
                continue  # pragma: no cover
            target = path.name
            content = read_text(path)
            if '+' in path.name:
                target = re.sub(
                    '\+([a-z]+)\+',
                    lambda m: '{' + m.groups()[0] + '}',
                    path.name
                ).format(**md)
            if target.endswith('_tmpl'):
                target = target[:-5]
                content = content.format(**md)
            write_text(outdir / target, content)
        else:
            target = outdir / path.name
            if target.exists():
                shutil.rmtree(str(target))
            shutil.copytree(str(path), str(target))
    del md['id']
    jsonlib.dump(md, outdir / 'metadata.json', indent=4)
521517
public function train(Dataset $dataset) : void
    {
        if (!$dataset instanceof Labeled) {
            throw new InvalidArgumentException('This Estimator requires a'
                . ' Labeled training set.');
        }

        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        $classes = $dataset->possibleOutcomes();

        $this->classes = $classes;
        $this->weights = array_fill_keys($classes, 0);

        $this->means = $this->variances = array_fill_keys($classes, []);

        foreach ($dataset->stratify() as $class => $stratum) {
            $means = $variances = [];

            foreach ($stratum->columns() as $values) {
                [$mean, $variance] = Stats::meanVar($values);

                $means[] = $mean;
                $variances[] = $variance ?: EPSILON;
            }

            $this->means[$class] = $means;
            $this->variances[$class] = $variances;

            $this->weights[$class] += $stratum->numRows();
        }

        if ($this->fitPriors) {
            $this->priors = [];

            $total = array_sum($this->weights) ?: EPSILON;

            foreach ($this->weights as $class => $weight) {
                $this->priors[$class] = log($weight / $total);
            }
        }
    }
380494
public function execute(DataSetInterface $dataSet)
    {
        $addressKey = $dataSet[CompanyUnitAddressLabelRelationDataSet::ADDRESS_KEY];
        if (!isset($this->idCompanyUnitAddressCache[$addressKey])) {
            $companyUnitAddressQuery = new SpyCompanyUnitAddressQuery();
            $idCompanyUnitAddress = $companyUnitAddressQuery
                ->select(SpyCompanyUnitAddressTableMap::COL_ID_COMPANY_UNIT_ADDRESS)
                ->findOneByKey($addressKey);

            if (!$idCompanyUnitAddress) {
                throw new CompanyUnitAddressNotFoundException(sprintf('Could not find CompanyUnitAddress with key "%s".', $addressKey));
            }

            $this->idCompanyUnitAddressCache[$addressKey] = $idCompanyUnitAddress;
        }

        $dataSet[CompanyUnitAddressLabelRelationDataSet::ID_COMPANY_UNIT_ADDRESS] = $this->idCompanyUnitAddressCache[$addressKey];
    }
780448
public static function isUniqueIdValid($uniqueId, $class)
    {
        if (!$uniqueId) {
            throw new InvalidArgumentException(sprintf(
                __('For the service provider [%s], the container unique ID cannot be empty.', 'fulcrum'),
                $class
            ));
        }

        if (!is_string($uniqueId)) {
            throw new InvalidArgumentException(sprintf(
                __('For the service provider [%s], the container unique ID must be a string. %s given.', 'fulcrum'),
                $class,
                gettype($uniqueId)
            ));
        }

        return true;
    }
1364826
public static function get_structured_data(\stored_file $dataset) {

        if ($dataset->get_filearea() !== 'unlabelled') {
            throw new \coding_exception('Sorry, only support for unlabelled data');
        }

        $rh = $dataset->get_content_file_handle();

        // Skip dataset info.
        fgets($rh);
        fgets($rh);

        $calculations = array();

        $headers = fgetcsv($rh);
        // Get rid of the sampleid column name.
        array_shift($headers);

        while ($columns = fgetcsv($rh)) {
            $uniquesampleid = array_shift($columns);

            // Unfortunately fgetcsv does not respect line's var types.
            $calculations[$uniquesampleid] = array_map(function($value) {

                if ($value === '') {
                    // We really want them as null because converted to float become 0
                    // and we need to treat the values separately.
                    return null;
                } else if (is_numeric($value)) {
                    return floatval($value);
                }
                return $value;
            }, array_combine($headers, $columns));
        }

        return $calculations;
    }