Gets training examples separated by documents
326697
protected boolean trainCorpus(List<List<Acronym>> corpus, List<Map<String, String>> trueLabels){
		boolean converge = false;
		
		// Unsupervised
		trueLabels = null;

		initModelParamsAndCounters();

		int n = corpus.size();

		int c = 1;
		Double change; 

		LOG.debug("Training abbreviations...");
		while(!converge){

			for (int i = 0; i < n; i++) {
				List<Acronym> docAcronyms = corpus.get(i);
				Map<String, String> docTrueLabels = null;
				if(trueLabels != null)
					docTrueLabels = trueLabels.get(i);
				int m = docAcronyms.size();
				for (int j = 0; j < m; j++) {
					Acronym currAcronym = docAcronyms.get(j);
					if(trueLabels != null)
						expectationStep(currAcronym, docTrueLabels.get(currAcronym._shortForm));
					else
						expectationStep(currAcronym, null);
				}
			}

			change = maximizationStep();
			LOG.debug("step {}...",c);

			c++;
			if(c > MAX_ITERATIONS){
				System.out.println("\n\tTraining stopped after "+(c-1)+" iterations with final change: "+change);
				converge = true;
			}
			if(change.compareTo(CHANGE_THRESHOLD) < 0){
				LOG.debug("Training converged in "+(c-1)+" iterations.");
				converge = true;
			}
		}
		saveModelParams();
		return true;
	}
691242
def get_docs_from_split(session, candidate_classes, split):
    """"""
    # Only grab the docs containing candidates from the given split.
    sub_query = session.query(Candidate.id).filter(Candidate.split == split).subquery()
    split_docs = set()
    for candidate_class in candidate_classes:
        split_docs.update(
            cand.document
            for cand in session.query(candidate_class)
            .filter(candidate_class.id.in_(sub_query))
            .all()
        )
    return split_docs
1180998
def train(args):
    """"""
    vocab, row, col, counts = get_train_data(args)
    model = GloVe(token_to_idx=vocab.token_to_idx, output_dim=args.emsize,
                  dropout=args.dropout, x_max=args.x_max, alpha=args.alpha,
                  weight_initializer=mx.init.Uniform(scale=1 / args.emsize))
    context = get_context(args)
    model.initialize(ctx=context)
    if not args.no_hybridize:
        model.hybridize(static_alloc=not args.no_static_alloc)

    optimizer_kwargs = dict(learning_rate=args.lr, eps=args.adagrad_eps)
    params = list(model.collect_params().values())
    try:
        trainer = mx.gluon.Trainer(params, 'groupadagrad', optimizer_kwargs)
    except ValueError:
        logging.warning('MXNet <= v1.3 does not contain '
                        'GroupAdaGrad support. Falling back to AdaGrad')
        trainer = mx.gluon.Trainer(params, 'adagrad', optimizer_kwargs)

    index_dtype = 'int32'
    if counts.shape[0] >= np.iinfo(np.int32).max:
        index_dtype = 'int64'
        logging.info('Co-occurrence matrix is large. '
                     'Using int64 to represent sample indices.')
    indices = mx.nd.arange(counts.shape[0], dtype=index_dtype)
    for epoch in range(args.epochs):
        # Logging variables
        log_wc = 0
        log_start_time = time.time()
        log_avg_loss = 0

        mx.nd.shuffle(indices, indices)  # inplace shuffle
        bs = args.batch_size
        num_batches = indices.shape[0] // bs
        for i in range(num_batches):
            batch_indices = indices[bs * i:bs * (i + 1)]
            ctx = context[i % len(context)]
            batch_row = row[batch_indices].as_in_context(ctx)
            batch_col = col[batch_indices].as_in_context(ctx)
            batch_counts = counts[batch_indices].as_in_context(ctx)
            with mx.autograd.record():
                loss = model(batch_row, batch_col, batch_counts)
                loss.backward()

            if len(context) == 1 or (i + 1) % len(context) == 0:
                trainer.step(batch_size=1)

            # Logging
            log_wc += loss.shape[0]
            log_avg_loss += loss.mean().as_in_context(context[0])
            if (i + 1) % args.log_interval == 0:
                # Forces waiting for computation by computing loss value
                log_avg_loss = log_avg_loss.asscalar() / args.log_interval
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'.format(
                                 epoch, i + 1, num_batches, log_avg_loss,
                                 wps / 1000, log_wc / 1000))
                log_dict = dict(
                    global_step=epoch * len(indices) + i * args.batch_size,
                    epoch=epoch, batch=i + 1, loss=log_avg_loss,
                    wps=wps / 1000)
                log(args, log_dict)

                log_start_time = time.time()
                log_avg_loss = 0
                log_wc = 0

            if args.eval_interval and (i + 1) % args.eval_interval == 0:
                with print_time('mx.nd.waitall()'):
                    mx.nd.waitall()
                with print_time('evaluate'):
                    evaluate(args, model, vocab, i + num_batches * epoch)

    # Evaluate
    with print_time('mx.nd.waitall()'):
        mx.nd.waitall()
    with print_time('evaluate'):
        evaluate(args, model, vocab, num_batches * args.epochs,
                 eval_analogy=not args.no_eval_analogy)

    # Save params
    with print_time('save parameters'):
        model.save_parameters(os.path.join(args.logdir, 'glove.params'))
332807
def _generate_examples(self, filepath):
    
    # Simultaneously iterating through the different data sets in the hdf5
    # file will be slow with a single file. Instead, we first load everything
    # into memory before yielding the samples.
    image_array, values_array = _load_data(filepath)

    # We need to calculate the class labels from the float values in the file.
    labels_array = np.zeros_like(values_array, dtype=np.int64)
    for i in range(values_array.shape[1]):
      labels_array[:, i] = _discretize(values_array[:, i])  # pylint: disable=unsupported-assignment-operation

    for image, labels, values in moves.zip(image_array, labels_array,
                                           values_array):
      yield {
          "image": image,
          "label_floor_hue": labels[0],
          "label_wall_hue": labels[1],
          "label_object_hue": labels[2],
          "label_scale": labels[3],
          "label_shape": labels[4],
          "label_orientation": labels[5],
          "value_floor_hue": values[0],
          "value_wall_hue": values[1],
          "value_object_hue": values[2],
          "value_scale": values[3],
          "value_shape": values[4],
          "value_orientation": values[5],
      }
285860
def get_docs(self, vocab):
        """"""
        for string in self.strings:
            vocab[string]
        orth_col = self.attrs.index(ORTH)
        for tokens, spaces in zip(self.tokens, self.spaces):
            words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
            doc = Doc(vocab, words=words, spaces=spaces)
            doc = doc.from_array(self.attrs, tokens)
            yield doc
899011
def build_iters(data_dir, max_records, q, horizon, splits, batch_size):
    
    # Read in data as numpy array
    df = pd.read_csv(os.path.join(data_dir, "electricity.txt"), sep=",", header=None)
    feature_df = df.iloc[:, :].astype(float)
    x = feature_df.as_matrix()
    x = x[:max_records] if max_records else x

    # Construct training examples based on horizon and window
    x_ts = np.zeros((x.shape[0] - q, q, x.shape[1]))
    y_ts = np.zeros((x.shape[0] - q, x.shape[1]))
    for n in range(x.shape[0]):
        if n + 1 < q:
            continue
        elif n + 1 + horizon > x.shape[0]:
            continue
        else:
            y_n = x[n + horizon, :]
            x_n = x[n + 1 - q:n + 1, :]
        x_ts[n-q] = x_n
        y_ts[n-q] = y_n

    # Split into training and testing data
    training_examples = int(x_ts.shape[0] * splits[0])
    valid_examples = int(x_ts.shape[0] * splits[1])
    x_train, y_train = x_ts[:training_examples], \
                       y_ts[:training_examples]
    x_valid, y_valid = x_ts[training_examples:training_examples + valid_examples], \
                       y_ts[training_examples:training_examples + valid_examples]
    x_test, y_test = x_ts[training_examples + valid_examples:], \
                     y_ts[training_examples + valid_examples:]

    #build iterators to feed batches to network
    train_iter = mx.io.NDArrayIter(data=x_train,
                                   label=y_train,
                                   batch_size=batch_size)
    val_iter = mx.io.NDArrayIter(data=x_valid,
                                 label=y_valid,
                                 batch_size=batch_size)
    test_iter = mx.io.NDArrayIter(data=x_test,
                                  label=y_test,
                                  batch_size=batch_size)
    return train_iter, val_iter, test_iter
447011
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'examples' in _dict:
            args['examples'] = [
                TrainingExample._from_dict(x) for x in (_dict.get('examples'))
            ]
        return cls(**args)
961167
def iter_documents(self, fileids=None, categories=None, _destroy=False):
        
        doc_ids = self._filter_ids(fileids, categories)
        for doc in imap(self.get_document, doc_ids):
            yield doc
            if _destroy:
                doc.destroy()
56360
def get_example_features(example):
  """"""
  return (example.features.feature if isinstance(example, tf.train.Example)
          else example.context.feature)
299916
def readTrainingData(file_locations, GROUP_LABEL):
    '''
    
    '''
    class Mock(object):
        pass
    mock_module = Mock()
    mock_module.PARENT_LABEL = GROUP_LABEL
    for location in file_locations:
        with open(location) as f:
            tree = etree.parse(f)

        xml = tree.getroot()
        for each in data_prep_utils.TrainingData(xml, mock_module):
            yield each
441580
def get_example():
        """"""
        example = dict()
        for section_key, section in Metadata.MAIN_SECTIONS.items():
            example[section_key] = section.EXAMPLE.copy()

        return example
548609
def train_phrases(paths, out='data/bigram_model.phrases', tokenizer=word_tokenize, **kwargs):
    
    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    # Change to use less memory. Default is 40m.
    kwargs = {
        'max_vocab_size': 40000000,
        'threshold': 8.
    }.update(kwargs)

    print('Training bigrams...')
    bigram = Phrases(_phrase_doc_stream(paths, n, tokenizer=word_tokenize), **kwargs)

    print('Saving...')
    bigram.save(out)
147291
def train_doc2vec(paths, out='data/model.d2v', tokenizer=word_tokenize, sentences=False, **kwargs):
    
    kwargs = {
        'size': 400,
        'window': 8,
        'min_count': 2,
        'workers': 8
    }.update(kwargs)

    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    print('Training doc2vec model...')
    m = Doc2Vec(_doc2vec_doc_stream(paths, n, tokenizer=tokenizer, sentences=sentences), **kwargs)

    print('Saving...')
    m.save(out)
1366333
def vectorize( self, docs ):
        '''
        
        '''

        if type(docs) == dict:
            docs = docs.items()

        if self.model == None:
            self.train(docs)

        asset_id2vector = {}

        unfound = []
        for item in docs:
            ## iterate through the items in docs and check if any are already in the model.
            asset_id, _ = item
            label = 'DOC_' + str(asset_id)
            if label in self.model:
                asset_id2vector.update({asset_id: self.model['DOC_' + str(asset_id)]})
            else:
                unfound.append(item)

        if len(unfound) > 0:
            ## for all assets not in the model, update the model and then get their sentence vectors.
            sentences = [self._gen_sentence(item) for item in unfound]
            self.update_model(sentences, train=self.stream_train)
            asset_id2vector.update({item[0]: self.model['DOC_' + str(item[0])] for item in unfound})

        return asset_id2vector
536156
def train(klass, text)
      th = TextHash.new(text)
      th.each { |word, count|
        @storage.incr_word_count klass, word, count
        yield word, count if block_given?
      }
      @storage.incr_total_word_count klass, th.word_count
      doccount = (text.kind_of? Array) ? text.length : 1
      @storage.incr_doc_count klass, doccount
      @classnames << klass unless @classnames.include? klass
      # cache is now dirty of these vars
      @doc_count_totals = nil
      @vocab_sizes = nil
      th
    end
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
983880
protected function getDocumentArray(TrainingSet $documents, FeatureFactoryInterface $ff)
    {
        $docs = array();
        foreach ($documents as $d) {
            $docs[] = $ff->getFeatureArray('',$d);
        }

        return $docs;
    }