Read training from previously built training data file object

        Arguments:

        training_file -- file object containing the training data
487171
def readTraining(self, training_file):
        '''
        
        '''

        logger.info('reading training from file')
        training_pairs = json.load(training_file,
                                   cls=serializer.dedupe_decoder)
        self.markPairs(training_pairs)
1237364
def train_and_save(self, model_filename, cannon_data_filename, 
        clobber=False, **kwargs):
        

        if any(map(os.path.exists,
            (model_filename, cannon_data_filename))) and not clobber:
            raise IOError("output file already exists")

        trained = self.train_global(**kwargs)
        with open(cannon_data_filename, "wb") as fp:
            pickle.dump(trained, fp, -1)

        self._configuration["model_grid"]["cannon_data"] = cannon_data_filename
        logger.info("Cannon coefficients pickled to {}".format(
            cannon_data_filename))

        self.save(model_filename, clobber)

        return True
770810
def get_traindata(self) -> np.ndarray:
        
        traindata = None
        for key, value in self.data.items():
            if key not in ['__header__', '__version__', '__globals__']:
                if traindata is None:
                    traindata = value[np.where(value[:, 4] != 0)]
                else:
                    traindata = np.concatenate((traindata, value[np.where(value[:, 4] != 0)]))
        return traindata
264308
def train_batch_gen(self) -> Iterator:
        

        if len(self.train_fns) == 0:
            raise PersephoneException("""No training data available; cannot
                                       generate training batches.""")

        # Create batches of batch_size and shuffle them.
        fn_batches = self.make_batches(self.train_fns)


        if self.rand:
            random.shuffle(fn_batches)

        for fn_batch in fn_batches:
            logger.debug("Batch of training filenames: %s",
                          pprint.pformat(fn_batch))
            yield self.load_batch(fn_batch)
        else:
            raise StopIteration
965436
def get_training_data(batch_size):
    
    return gluon.data.DataLoader(
        CIFAR10(train=True, transform=transformer),
        batch_size=batch_size, shuffle=True, last_batch='discard')
1149791
def write_training_metrics(self):
        
        with open(self.path, 'w') as file:
            writer = csv.writer(file)
            writer.writerow(FIELD_NAMES)
            for row in self.rows:
                writer.writerow(row)
1185778
public void train(String dataFile, int maxite, float c) throws IOException {

		fp = File.createTempFile("train-features", null, new File("./tmp/"));

		buildInstanceList(dataFile);

		LabelAlphabet postagAlphabet = factory.buildLabelAlphabet("postag");


		SFGenerator generator = new SFGenerator();
		Linear[] models = new Linear[postagAlphabet.size()];

		for (int i = 0; i < postagAlphabet.size(); i++) {
			String pos = postagAlphabet.lookupString(i);
			InstanceSet instset = readInstanceSet(pos);
			LabelAlphabet alphabet = factory.buildLabelAlphabet(pos);
			int ysize = alphabet.size();
			System.out.printf("Training with data: %s\n", pos);
			System.out.printf("Number of labels: %d\n", ysize);
			LinearMax solver = new LinearMax(generator, ysize);
			ZeroOneLoss loss = new ZeroOneLoss();
			Update update = new LinearMaxPAUpdate(loss);
			OnlineTrainer trainer = new OnlineTrainer(solver, update, loss,
					factory, maxite, c);
			models[i] = trainer.train(instset, null);
			instset = null;
			solver = null;
			loss = null;
			trainer = null;
			System.out.println();
		}
		factory.setStopIncrement(true);
		saveModels(modelfile, models,factory);

		fp.delete();
		fp = null;
	}
582582
def read_examples(input_files, batch_size, shuffle, num_epochs=None):
  """"""
  files = []
  for e in input_files:
    for path in e.split(','):
      files.extend(file_io.get_matching_files(path))
  thread_count = multiprocessing.cpu_count()

  # The minimum number of instances in a queue from which examples are drawn
  # randomly. The larger this number, the more randomness at the expense of
  # higher memory requirements.
  min_after_dequeue = 1000

  # When batching data, the queue's capacity will be larger than the batch_size
  # by some factor. The recommended formula is (num_threads + a small safety
  # margin). For now, we use a single thread for reading, so this can be small.
  queue_size_multiplier = thread_count + 3

  # Convert num_epochs == 0 -> num_epochs is None, if necessary
  num_epochs = num_epochs or None

  # Build a queue of the filenames to be read.
  filename_queue = tf.train.string_input_producer(files, num_epochs, shuffle)

  example_id, encoded_example = tf.TextLineReader().read_up_to(
      filename_queue, batch_size)

  if shuffle:
    capacity = min_after_dequeue + queue_size_multiplier * batch_size
    return tf.train.shuffle_batch(
        [example_id, encoded_example],
        batch_size,
        capacity,
        min_after_dequeue,
        enqueue_many=True,
        num_threads=thread_count)

  else:
    capacity = queue_size_multiplier * batch_size
    return tf.train.batch(
        [example_id, encoded_example],
        batch_size,
        capacity=capacity,
        enqueue_many=True,
        num_threads=thread_count)
952340
def writeTraining(self, file_obj):  # pragma: no cover
        

        json.dump(self.training_pairs,
                  file_obj,
                  default=serializer._to_json,
                  tuple_as_array=False,
                  ensure_ascii=True)
535429
def train():
    
    print(model)
    from_epoch = 0
    model.initialize(mx.init.Xavier(factor_type='out'), ctx=context)
    trainer_params = {'learning_rate': args.lr, 'wd': 0, 'eps': args.eps}
    trainer = gluon.Trainer(model.collect_params(), 'adagrad', trainer_params)
    if args.from_epoch:
        from_epoch = args.from_epoch
        checkpoint_name = '%s.%s'%(args.save, format(from_epoch - 1, '02d'))
        model.load_parameters(checkpoint_name)
        trainer.load_states('%s.state'%args.save)
        print('Loaded parameters from checkpoint %s'%(checkpoint_name))

    model.hybridize(static_alloc=True, static_shape=True)
    encoder_params = model.encoder.collect_params().values()
    embedding_params = list(model.embedding.collect_params().values())
    parallel_model = ParallelBigRNN(model, loss)
    parallel = Parallel(len(context), parallel_model)
    for epoch in range(from_epoch, args.epochs):
        sys.stdout.flush()
        total_L = 0.0
        start_epoch_time = time.time()
        start_log_interval_time = time.time()
        hiddens = [model.begin_state(batch_size=args.batch_size,
                                     func=mx.nd.zeros, ctx=ctx) for ctx in context]
        nbatch = 0
        has_next = True
        train_data_iter = iter(train_data)
        data, target, mask, sample = next(train_data_iter)

        while has_next:
            nbatch += 1
            hiddens = detach(hiddens)
            Ls = []
            for _, batch in enumerate(zip(data, target, mask, sample, hiddens)):
                parallel.put(batch)

            for _ in range(len(data)):
                hidden, ls = parallel.get()
                # hidden states are ordered by context id
                index = context.index(hidden[0].context)
                hiddens[index] = hidden
                Ls.append(ls)

            # prefetch the next batch of data
            try:
                data, target, mask, sample = next(train_data_iter)
            except StopIteration:
                has_next = False

            # rescale embedding grad
            for ctx in context:
                x = embedding_params[0].grad(ctx)
                x[:] *= args.batch_size
                encoder_grad = [p.grad(ctx) for p in encoder_params]
                # perform gradient clipping per ctx
                gluon.utils.clip_global_norm(encoder_grad, args.clip)

            trainer.step(len(context))

            total_L += sum([mx.nd.sum(L).asscalar() / args.bptt for L in Ls])

            if nbatch % args.log_interval == 0:
                cur_L = total_L / args.log_interval / len(context)
                ppl = math.exp(cur_L) if cur_L < 100 else float('inf')
                print('[Epoch %d Batch %d] loss %.2f, ppl %.2f, '
                      'throughput %.2f samples/s'
                      %(epoch, nbatch, cur_L, ppl,
                        train_batch_size*args.log_interval/(time.time()-start_log_interval_time)))
                total_L = 0.0
                start_log_interval_time = time.time()
                sys.stdout.flush()

        end_epoch_time = time.time()
        print('Epoch %d took %.2f seconds.'%(epoch, end_epoch_time - start_epoch_time))
        mx.nd.waitall()
        checkpoint_name = '%s.%s'%(args.save, format(epoch, '02d'))
        model.save_parameters(checkpoint_name)
        trainer.save_states('%s.state'%args.save)
1075426
def import_training_data(self,
            positive_corpus_file=os.path.join(os.path.dirname(__file__),
                "positive.txt"),
            negative_corpus_file=os.path.join(os.path.dirname(__file__),
                "negative.txt")
            ):
        

        positive_corpus = open(positive_corpus_file)
        negative_corpus = open(negative_corpus_file)

        # for line in positive_corpus: 
        #     self.training_data.append((line, True))

        # for line in negative_corpus: 
        #     self.training_data.append((line, False))

        # The following code works. Need to profile this to see if this is an
        # improvement over the code above. 
        positive_training_data = list(map(lambda x: (x, True), positive_corpus))
        negative_training_data = list(map(lambda x: (x, False), negative_corpus))
        self.training_data = positive_training_data + negative_training_data
714793
def train(self): 
        
        if not self.training_data: 
            self.import_training_data()
        training_feature_set = [(self.extract_features(line), label) 
                                    for (line, label) in self.training_data]
        self.classifier = nltk.NaiveBayesClassifier.train(training_feature_set)
8601
def training_data(self):
        

        data = pickle.load(open(os.path.join(self.repopath, 'training.pkl')))
        return data.keys(), data.values()
1313777
def load_training_data
      File.open(@training_file) {|file|
        @training_set = Nimbus::TrainingSet.new({}, {})
        file.each do |line|
          next if line.strip == ''
          data_feno, data_id, *snp_list = line.strip.split
          raise Nimbus::InputFileError, "Individual ##{data_id} from training set has no value for all #{@tree_SNP_total_count} SNPs" unless snp_list.size == @tree_SNP_total_count
          raise Nimbus::InputFileError, "There are individuals with no ID, please check data in training file." unless (!data_id.nil? && data_id.strip != '')
          raise Nimbus::InputFileError, "Individual ##{data_id} has no fenotype value, please check data in training file." unless (!data_feno.nil? && data_feno.strip != '')
          raise Nimbus::InputFileError, "Individual ##{data_id} has invalid class (not in [#{classes*', '}]), please check data in training file." unless (@classes.nil? || @classes.include?(data_feno))

          data_feno = (@classes ? data_feno.to_s : data_feno.to_f)
          @training_set.individuals[data_id.to_i] = Nimbus::Individual.new(data_id.to_i, data_feno, snp_list.map{|snp| snp.to_i})
          @training_set.ids_fenotypes[data_id.to_i] = data_feno
        end
      }
    end
890374
private void docTrain() throws IOException {
    // load training parameters file
    final String paramFile = this.parsedArguments.getString("params");
    final TrainingParameters params = IOUtils.loadTrainingParameters(paramFile);
    String outModel = null;
    if (params.getSettings().get("OutputModel") == null
        || params.getSettings().get("OutputModel").length() == 0) {
      outModel = Files.getNameWithoutExtension(paramFile) + ".bin";
      params.put("OutputModel", outModel);
    } else {
      outModel = Flags.getModel(params);
    }
    final DocumentClassifierTrainer docTrainer = new DocumentClassifierTrainer(
        params);
    final DocumentClassifierModel trainedModel = docTrainer.train(params);
    CmdLineUtil.writeModel("ixa-pipe-ml", new File(outModel), trainedModel);
  }
443586
def from_raw(self, file_names=None, **kwargs):
        
        # This function only loads one test at a time (but could contain several
        # files). The function from_res() also implements loading several
        # datasets (using list of lists as input).

        if file_names:
            self.file_names = file_names

        if not isinstance(file_names, (list, tuple)):
            self.file_names = [file_names, ]

        # file_type = self.tester
        raw_file_loader = self.loader
        set_number = 0
        test = None
        counter = 0
        self.logger.debug("start iterating through file(s)")
        for f in self.file_names:
            self.logger.debug("loading raw file:")
            self.logger.debug(f"{f}")
            new_tests = raw_file_loader(f, **kwargs)
            if new_tests:
                if test is not None:
                    self.logger.debug("continuing reading files...")
                    _test = self._append(test[set_number], new_tests[set_number])
                    if not _test:
                        self.logger.warning(f"EMPTY TEST: {f}")
                        continue
                    test[set_number] = _test
                    self.logger.debug("added this test - started merging")
                    for j in range(len(new_tests[set_number].raw_data_files)):
                        raw_data_file = new_tests[set_number].raw_data_files[j]
                        file_size = new_tests[set_number].raw_data_files_length[j]
                        test[set_number].raw_data_files.append(raw_data_file)
                        test[set_number].raw_data_files_length.append(file_size)
                        counter += 1
                        if counter > 10:
                            self.logger.debug("ERROR? Too many files to merge")
                            raise ValueError("Too many files to merge - "
                                             "could be a p2-p3 zip thing")
                else:
                    self.logger.debug("getting data from first file")
                    if new_tests[set_number].no_data:
                        self.logger.debug("NO DATA")
                    else:
                        test = new_tests
            else:
                self.logger.debug("NOTHING LOADED")

        self.logger.debug("finished loading the raw-files")

        test_exists = False
        if test:
            if test[0].no_data:
                self.logging.debug("the first dataset (or only dataset) loaded from the raw data file is empty")
            else:
                test_exists = True

        if test_exists:
            if not prms.Reader.sorted_data:
                self.logger.debug("sorting data")
                test[set_number] = self._sort_data(test[set_number])

            self.datasets.append(test[set_number])
        else:
            self.logger.warning("No new datasets added!")
        self.number_of_datasets = len(self.datasets)
        self.status_datasets = self._validate_datasets()
        self._invent_a_name()
        return self
380297
def load_learner(path:PathOrStr, file:PathLikeOrBinaryStream='export.pkl', test:ItemList=None, **db_kwargs):
    ""
    source = Path(path)/file if is_pathlike(file) else file
    state = torch.load(source, map_location='cpu') if defaults.device == torch.device('cpu') else torch.load(source)
    model = state.pop('model')
    src = LabelLists.load_state(path, state.pop('data'))
    if test is not None: src.add_test(test)
    data = src.databunch(**db_kwargs)
    cb_state = state.pop('cb_state')
    clas_func = state.pop('cls')
    res = clas_func(data, model, **state)
    res.callback_fns = state['callback_fns'] #to avoid duplicates
    res.callbacks = [load_callback(c,s, res) for c,s in cb_state.items()]
    return res