Train the language with all the given documents.

@param dataset
an array with hashes of the format:
{word1:count1, word2:count2,...}
each object represents the a sentence (it should be tokenized in advance).
485740
function(dataset) {

		// calculate counts for equation (3):
		var mapWordToTotalCount = {};
		var totalNumberOfWordsInDataset = 0;
		for (var i in dataset) {
			var datum = dataset[i];
			var totalPerDatum = 0;
			
			// for each input sentence, count the total number of words in it:
			for (var word in datum) {
				mapWordToTotalCount[word] |= 0;
				mapWordToTotalCount[word] += datum[word];
				totalPerDatum += datum[word];
			}
			datum["_total"] = totalPerDatum;
			totalNumberOfWordsInDataset += totalPerDatum;
		}
		mapWordToTotalCount["_total"] = totalNumberOfWordsInDataset;
		
		this.dataset = dataset;
		this.mapWordToTotalCount = mapWordToTotalCount;
		
		// calculate smoothing factor for equation (3):
		var mapWordToSmoothingFactor = {};
		for (var word in mapWordToTotalCount) {
			mapWordToSmoothingFactor[word] = 
				(1-this.smoothingCoefficient) * this.mapWordToTotalCount[word] / this.mapWordToTotalCount["_total"];
		}
		this.mapWordToSmoothingFactor = mapWordToSmoothingFactor;
		
		this.globalSmoothingFactor = (1/totalNumberOfWordsInDataset)  // a global smoother, for totally unseen words.
	}
281042
public function train(Dataset $dataset) : void
    {
        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        $columns = $dataset->columns();

        $this->medians = $this->mads = [];

        foreach ($columns as $column => $values) {
            [$median, $mad] = Stats::medMad($values);

            $this->medians[$column] = $median;
            $this->mads[$column] = $mad;
        }
    }
1281025
public function train(Dataset $dataset) : void
    {
        if (!$dataset instanceof Labeled) {
            throw new InvalidArgumentException('This Estimator requires a'
                . ' Labeled training set.');
        }

        $classes = $dataset->possibleOutcomes();

        $this->classes = $classes;
        $this->weights = array_fill_keys($classes, 0);

        $this->counts = $this->probs = array_fill_keys(
            $classes,
            array_fill(0, $dataset->numColumns(), [])
        );

        $this->partial($dataset);
    }
1332230
public void setDatasets(java.util.Collection<Dataset> datasets) {
        if (datasets == null) {
            this.datasets = null;
            return;
        }

        this.datasets = new com.amazonaws.internal.SdkInternalList<Dataset>(datasets);
    }
802855
public function rank(Dataset $dataset) : array
    {
        if ($this->tree->bare()) {
            throw new RuntimeException('The learner has not'
                . ' been trained.');
        }
        
        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        return array_map([self::class, 'localOutlierFactor'], $dataset->samples());
    }
223139
def setDataset(self, dataset):
        
        self._dataset = dataset
        
        # setup the tooltip
        tip = []
        tip.append('<b>%s</b>' % dataset.name())
        for value in dataset.values():
            value_text = []
            for key, val in sorted(value.items()):
                if val == dataset.name():
                    continue
                
                axis = self.axis(key)
                if axis and axis.labelFormat():
                    val = axis.labelFormat().format(val)
                value_text.append('%s: %s' % (key, val))
            
            tip.append('<p>%s</p>' % ', '.join(value_text))
        self.setToolTip(''.join(tip))
1363234
public void train(JavaRDD<String> corpusRDD) throws Exception {
        log.info("Start training ...");

        if (workers > 0)
            corpusRDD.repartition(workers);

        // SparkContext
        final JavaSparkContext sc = new JavaSparkContext(corpusRDD.context());

        // Pre-defined variables
        Map<String, Object> tokenizerVarMap = getTokenizerVarMap();
        Map<String, Object> word2vecVarMap = getWord2vecVarMap();

        // Variables to fill in train
        final JavaRDD<AtomicLong> sentenceWordsCountRDD;
        final JavaRDD<List<VocabWord>> vocabWordListRDD;
        final JavaPairRDD<List<VocabWord>, Long> vocabWordListSentenceCumSumRDD;
        final VocabCache<VocabWord> vocabCache;
        final JavaRDD<Long> sentenceCumSumCountRDD;
        int maxRep = 1;

        // Start Training //
        //////////////////////////////////////
        log.info("Tokenization and building VocabCache ...");
        // Processing every sentence and make a VocabCache which gets fed into a LookupCache
        Broadcast<Map<String, Object>> broadcastTokenizerVarMap = sc.broadcast(tokenizerVarMap);
        TextPipeline pipeline = new TextPipeline(corpusRDD, broadcastTokenizerVarMap);
        pipeline.buildVocabCache();
        pipeline.buildVocabWordListRDD();

        // Get total word count and put into word2vec variable map
        word2vecVarMap.put("totalWordCount", pipeline.getTotalWordCount());

        // 2 RDDs: (vocab words list) and (sentence Count).Already cached
        sentenceWordsCountRDD = pipeline.getSentenceCountRDD();
        vocabWordListRDD = pipeline.getVocabWordListRDD();

        // Get vocabCache and broad-casted vocabCache
        Broadcast<VocabCache<VocabWord>> vocabCacheBroadcast = pipeline.getBroadCastVocabCache();
        vocabCache = vocabCacheBroadcast.getValue();

        log.info("Vocab size: {}", vocabCache.numWords());

        //////////////////////////////////////
        log.info("Building Huffman Tree ...");
        // Building Huffman Tree would update the code and point in each of the vocabWord in vocabCache
        /*
        We don't need to build tree here, since it was built earlier, at TextPipeline.buildVocabCache() call.
        
        Huffman huffman = new Huffman(vocabCache.vocabWords());
        huffman.build();
        huffman.applyIndexes(vocabCache);
        */
        //////////////////////////////////////
        log.info("Calculating cumulative sum of sentence counts ...");
        sentenceCumSumCountRDD = new CountCumSum(sentenceWordsCountRDD).buildCumSum();

        //////////////////////////////////////
        log.info("Mapping to RDD(vocabWordList, cumulative sentence count) ...");
        vocabWordListSentenceCumSumRDD =
                        vocabWordListRDD.zip(sentenceCumSumCountRDD).setName("vocabWordListSentenceCumSumRDD");

        /////////////////////////////////////
        log.info("Broadcasting word2vec variables to workers ...");
        Broadcast<Map<String, Object>> word2vecVarMapBroadcast = sc.broadcast(word2vecVarMap);
        Broadcast<double[]> expTableBroadcast = sc.broadcast(expTable);



        /////////////////////////////////////
        log.info("Training word2vec sentences ...");
        FlatMapFunction firstIterFunc =
                        new FirstIterationFunction(word2vecVarMapBroadcast, expTableBroadcast, vocabCacheBroadcast);
        @SuppressWarnings("unchecked")
        JavaRDD<Pair<VocabWord, INDArray>> indexSyn0UpdateEntryRDD =
                        vocabWordListSentenceCumSumRDD.mapPartitions(firstIterFunc).map(new MapToPairFunction());

        // Get all the syn0 updates into a list in driver
        List<Pair<VocabWord, INDArray>> syn0UpdateEntries = indexSyn0UpdateEntryRDD.collect();

        // Instantiate syn0
        INDArray syn0 = Nd4j.zeros(vocabCache.numWords(), layerSize);

        // Updating syn0 first pass: just add vectors obtained from different nodes
        log.info("Averaging results...");
        Map<VocabWord, AtomicInteger> updates = new HashMap<>();
        Map<Long, Long> updaters = new HashMap<>();
        for (Pair<VocabWord, INDArray> syn0UpdateEntry : syn0UpdateEntries) {
            syn0.getRow(syn0UpdateEntry.getFirst().getIndex()).addi(syn0UpdateEntry.getSecond());

            // for proper averaging we need to divide resulting sums later, by the number of additions
            if (updates.containsKey(syn0UpdateEntry.getFirst())) {
                updates.get(syn0UpdateEntry.getFirst()).incrementAndGet();
            } else
                updates.put(syn0UpdateEntry.getFirst(), new AtomicInteger(1));

            if (!updaters.containsKey(syn0UpdateEntry.getFirst().getVocabId())) {
                updaters.put(syn0UpdateEntry.getFirst().getVocabId(), syn0UpdateEntry.getFirst().getAffinityId());
            }
        }

        // Updating syn0 second pass: average obtained vectors
        for (Map.Entry<VocabWord, AtomicInteger> entry : updates.entrySet()) {
            if (entry.getValue().get() > 1) {
                if (entry.getValue().get() > maxRep)
                    maxRep = entry.getValue().get();
                syn0.getRow(entry.getKey().getIndex()).divi(entry.getValue().get());
            }
        }

        long totals = 0;

        log.info("Finished calculations...");


        vocab = vocabCache;
        InMemoryLookupTable<VocabWord> inMemoryLookupTable = new InMemoryLookupTable<VocabWord>();
        Environment env = EnvironmentUtils.buildEnvironment();
        env.setNumCores(maxRep);
        env.setAvailableMemory(totals);
        update(env, Event.SPARK);
        inMemoryLookupTable.setVocab(vocabCache);
        inMemoryLookupTable.setVectorLength(layerSize);
        inMemoryLookupTable.setSyn0(syn0);
        lookupTable = inMemoryLookupTable;
        modelUtils.init(lookupTable);
    }
394713
@Override
    public void fit(DataSet dataSet) {
        featureStats = (S) newBuilder().addFeatures(dataSet).build();
        if (isFitLabel()) {
            labelStats = (S) newBuilder().addLabels(dataSet).build();
        }
    }
245127
public ClassificationMetrics validate(Map<Object, URI> datasets) {
        TrainingParameters trainingParameters = (TrainingParameters) knowledgeBase.getTrainingParameters();

        //build the testDataset
        Dataframe testDataset = Dataframe.Builder.parseTextFiles(
                datasets,
                AbstractTextExtractor.newInstance(trainingParameters.getTextExtractorParameters()),
                knowledgeBase.getConfiguration()
        );

        ClassificationMetrics vm = validate(testDataset);

        testDataset.close();

        return vm;
    }
417909
protected function update(Dataset $dataset) : void
    {
        foreach ($this->transformers as $transformer) {
            if ($transformer instanceof Elastic) {
                $transformer->update($dataset);

                if ($this->logger) {
                    $this->logger->info('Updated '
                        . Params::shortName($transformer));
                }
            }

            $dataset->apply($transformer);

            if ($this->logger) {
                $this->logger->info('Applied '
                    . Params::shortName($transformer));
            }
        }
    }
664596
public function train(Dataset $dataset) : void
    {
        if ($this->type() === self::CLASSIFIER or $this->type() === self::REGRESSOR) {
            if (!$dataset instanceof Labeled) {
                throw new InvalidArgumentException('This estimator requires a'
                    . ' labeled training set.');
            }
        }

        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        $p = (int) round($this->ratio * $dataset->numRows());

        $this->ensemble = [];

        Loop::run(function () use ($dataset, $p) {
            $pool = new DefaultPool($this->workers);

            $coroutines = [];

            for ($i = 0; $i < $this->estimators; $i++) {
                $estimator = clone $this->base;

                $subset = $dataset->randomSubsetWithReplacement($p);

                $task = new CallableTask(
                    [$this, '_train'],
                    [$estimator, $subset]
                );

                $coroutines[] = call(function () use ($pool, $task) {
                    return yield $pool->enqueue($task);
                });
            }

            $this->ensemble = yield all($coroutines);
            
            return yield $pool->shutdown();
        });
    }
877182
def convert_to_sequences(dataset, vocab):
    
    start = time.time()
    dataset_vocab = map(lambda x: (x, vocab), dataset)
    with mp.Pool() as pool:
        # Each sample is processed in an asynchronous manner.
        output = pool.map(get_sequence, dataset_vocab)
    end = time.time()
    logging.info('Done! Sequence conversion Time={:.2f}s, #Sentences={}'
                 .format(end - start, len(dataset)))
    return output
1294403
def read_dataset(args, dataset):
    
    path = os.path.join(vars(args)[dataset])
    logger.info('reading data from {}'.format(path))
    examples = [line.strip().split('\t') for line in open(path)]
    if args.max_num_examples > 0:
        examples = examples[:args.max_num_examples]
    # NOTE: assume data has been tokenized
    dataset = gluon.data.SimpleDataset([(e[0], e[1], LABEL_TO_IDX[e[2]]) for e in examples])
    dataset = dataset.transform(lambda s1, s2, label: (
        ['NULL'] + s1.lower().split(),
        ['NULL'] + s2.lower().split(), label),
                                lazy=False)
    logger.info('read {} examples'.format(len(dataset)))
    return dataset
489413
def train_language_model_from_dataset(self, dataset_id, name, token=None, url=API_TRAIN_LANGUAGE_MODEL):
        
        auth = 'Bearer ' + self.check_for_token(token)
        dummy_files = {'name': (None, name), 'datasetId':(None, dataset_id)}
        h = {'Authorization': auth, 'Cache-Control':'no-cache'}
        the_url = url
        r = requests.post(the_url, headers=h, files=dummy_files)

        return r
1246333
public function train(Dataset $dataset) : void
    {
        if (!$dataset instanceof Labeled) {
            throw new InvalidArgumentException('This Estimator requires a'
                . ' Labeled training set.');
        }

        DatasetIsCompatibleWithEstimator::check($dataset, $this);

        if ($this->logger) {
            $this->logger->info('Learner init ' . Params::stringify([
                'base' => $this->base,
                'metric' => $this->metric,
                'validator' => $this->validator,
                'workers' => $this->workers,
            ]));
        }

        $results = [];

        Loop::run(function () use (&$results, $dataset) {
            $pool = new DefaultPool($this->workers);

            $coroutines = [];

            foreach ($this->combinations as $params) {
                $estimator = new $this->base(...$params);

                $task = new CallableTask(
                    [$this, 'score'],
                    [$estimator, $dataset]
                );

                $coroutines[] = call(function () use ($pool, $task, $params) {
                    $score = yield $pool->enqueue($task);

                    if ($this->logger) {
                        $constructor = array_combine($this->args, $params) ?: [];
                            
                        $this->logger->info('Test complete: ' . Params::stringify([
                            Params::shortName($this->metric) => $score,
                        ]) . ' Params: ' . Params::stringify($constructor));
                    }

                    return [$score, $params];
                });
            }

            $results = yield all($coroutines);
            
            return yield $pool->shutdown();
        });

        $bestScore = -INF;
        $bestParams = [];

        foreach ($results as [$score, $params]) {
            if ($score > $bestScore) {
                $bestScore = $score;
                $bestParams = $params;
            }
        }

        $this->best = [
            'score' => $bestScore,
            'params' => $bestParams,
        ];

        if ($this->logger) {
            $constructor = array_combine($this->args, $bestParams) ?: [];

            $this->logger->info('Best params: ' . Params::stringify($constructor));
        }

        $estimator = new $this->base(...$bestParams);

        $estimator->train($dataset);

        $this->estimator = $estimator;

        if ($this->logger) {
            $this->logger->info('Training complete');
        }
    }
1096797
protected static void printDataSet(DataSet dataSet) {
        final String SPACER = "| \t |";

        if (dataSet == null) System.out.println("DataSet is null");
        if (dataSet.getRowCount() == 0) System.out.println("DataSet is empty");

        List<DataColumn> dataSetColumns = dataSet.getColumns();
        int colColunt = dataSetColumns.size();
        int rowCount = dataSet.getRowCount();

        System.out.println("********************************************************************************************************************************************************");
        for (int row = 0; row < rowCount; row++) {
            System.out.print(SPACER);
            for (int col= 0; col< colColunt; col++) {
                Object value = dataSet.getValueAt(row, col);
                String colId = dataSet.getColumnByIndex(col).getId();
                System.out.print(colId + ": " +  value);
                System.out.print(SPACER);
            }
            System.out.println("");
        }
        System.out.println("********************************************************************************************************************************************************");
    }
725103
public static MultiDataSet toMultiDataSet(DataSet dataSet) {
        INDArray f = dataSet.getFeatures();
        INDArray l = dataSet.getLabels();
        INDArray fMask = dataSet.getFeaturesMaskArray();
        INDArray lMask = dataSet.getLabelsMaskArray();

        INDArray[] fNew = f == null ? null : new INDArray[] {f};
        INDArray[] lNew = l == null ? null : new INDArray[] {l};
        INDArray[] fMaskNew = (fMask != null ? new INDArray[] {fMask} : null);
        INDArray[] lMaskNew = (lMask != null ? new INDArray[] {lMask} : null);

        return new org.nd4j.linalg.dataset.MultiDataSet(fNew, lNew, fMaskNew, lMaskNew);
    }