Load the model, set up the pipeline and train the parser.
196592
def main(model=None, output_dir=None, n_iter=15):
    """"""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    # We'll use the built-in dependency parser class, but we want to create a
    # fresh instance – just in case.
    if "parser" in nlp.pipe_names:
        nlp.remove_pipe("parser")
    parser = nlp.create_pipe("parser")
    nlp.add_pipe(parser, first=True)

    for text, annotations in TRAIN_DATA:
        for dep in annotations.get("deps", []):
            parser.add_label(dep)

    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "parser"]
    with nlp.disable_pipes(*other_pipes):  # only train parser
        optimizer = nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            # batch up the examples using spaCy's minibatch
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(texts, annotations, sgd=optimizer, losses=losses)
            print("Losses", losses)

    # test the trained model
    test_model(nlp)

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        test_model(nlp2)
376108
def main(model=None, new_model_name="animal", output_dir=None, n_iter=30):
    """"""
    random.seed(0)
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")
    # Add entity recognizer to model if it's not in the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pipe(ner)
    # otherwise, get it, so we can add labels to it
    else:
        ner = nlp.get_pipe("ner")

    ner.add_label(LABEL)  # add new entity label to entity recognizer
    # Adding extraneous labels shouldn't mess anything up
    ner.add_label("VEGETABLE")
    if model is None:
        optimizer = nlp.begin_training()
    else:
        optimizer = nlp.resume_training()
    move_names = list(ner.move_names)
    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
    with nlp.disable_pipes(*other_pipes):  # only train NER
        sizes = compounding(1.0, 4.0, 1.001)
        # batch up the examples using spaCy's minibatch
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            batches = minibatch(TRAIN_DATA, size=sizes)
            losses = {}
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(texts, annotations, sgd=optimizer, drop=0.35, losses=losses)
            print("Losses", losses)

    # test the trained model
    test_text = "Do you like horses?"
    doc = nlp(test_text)
    print("Entities in '%s'" % test_text)
    for ent in doc.ents:
        print(ent.label_, ent.text)

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.meta["name"] = new_model_name  # rename model
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        # Check the classes have loaded back consistently
        assert nlp2.get_pipe("ner").move_names == move_names
        doc2 = nlp2(test_text)
        for ent in doc2.ents:
            print(ent.label_, ent.text)
321393
def load(self):
        
        self.layer.ResetReading()
        for i in range(self.nfeatures):
            if self.__features[i] is None:
                self.__features[i] = self.layer[i]
1244961
public static void loadParser(String path) throws LoadModelException {
		if(parser==null){
			String file = path+parseModel;
			parser = new JointParser(file);
		}
	}
746925
public final ParserModel train(final TrainingParameters params,
      final TrainingParameters taggerParams,
      final TrainingParameters chunkerParams) {
    if (getParserFactory() == null) {
      throw new IllegalStateException(
          "The ParserFactory must be instantiated!!");
    }
    if (getTaggerFactory() == null) {
      throw new IllegalStateException(
          "The TaggerFactory must be instantiated!");
    }
    ParserModel trainedModel = null;
    ParserEvaluator parserEvaluator = null;
    try {
      trainedModel = ShiftReduceParser.train(this.lang, this.trainSamples,
          this.rules, params, this.parserFactory, taggerParams,
          this.taggerFactory, chunkerParams, this.chunkerFactory);
      final ShiftReduceParser parser = new ShiftReduceParser(trainedModel);
      parserEvaluator = new ParserEvaluator(parser);
      parserEvaluator.evaluate(this.testSamples);
    } catch (final IOException e) {
      System.err.println("IO error while loading training and test sets!");
      e.printStackTrace();
      System.exit(1);
    }
    System.out.println("Final Result: \n" + parserEvaluator.getFMeasure());
    return trainedModel;
  }
469404
def load_model(
            self, the_metamodel, filename, is_main_model, encoding='utf-8',
            add_to_local_models=True):
        

        if not self.local_models.has_model(filename):
            if self.all_models.has_model(filename):
                new_model = self.all_models.filename_to_model[filename]
            else:
                # print("LOADING {}".format(filename))
                # all models loaded here get their references resolved from the
                # root model
                new_model = the_metamodel.internal_model_from_file(
                    filename, pre_ref_resolution_callback=lambda
                    other_model: self.pre_ref_resolution_callback(other_model),
                    is_main_model=is_main_model, encoding=encoding)
                self.all_models.filename_to_model[filename] = new_model
            # print("ADDING {}".format(filename))
            if add_to_local_models:
                self.local_models.filename_to_model[filename] = new_model
        assert self.all_models.has_model(filename)  # to be sure...
        return self.all_models.filename_to_model[filename]
933651
def _load_model(self):
        
        super()._load_model()
        self.mujoco_robot = Sawyer()
        if self.has_gripper:
            self.gripper = gripper_factory(self.gripper_type)
            if not self.gripper_visualization:
                self.gripper.hide_visualization()
            self.mujoco_robot.add_gripper("right_hand", self.gripper)
748858
def on_train_begin(self, **kwargs):
        ""
        if not self.modules:
            self.modules = [m for m in flatten_model(self.learn.model)
                            if hasattr(m, 'weight')]
        self.hooks = Hooks(self.modules, self.hook)
992359
def on_train_begin(self, **kwargs:Any)->None:
        ""
        #Get a copy of the model params in FP32
        self.model_params, self.master_params = get_master(self.learn.layer_groups, self.flat_master)
        #Changes the optimizer so that the optimization step is done in FP32.
        new_opt = self.learn.opt.new_with_params(self.master_params)
        if self.opt is not None:
            self.opt.lr,self.opt.wd = self.learn.opt.lr,self.learn.opt.wd
            new_opt.load_state_dict(self.opt)
        self.learn.opt.opt = new_opt.opt
        self.noskip = 0
1132495
def initialize(self, training_info, model, environment, device):
        
        self.target_model = self.model_factory.instantiate(action_space=environment.action_space).to(device)
        self.target_model.load_state_dict(model.state_dict())
        self.target_model.eval()
785581
def maybe_load_model(savedir, container):
  """"""
  if savedir is None:
    return

  state_path = os.path.join(os.path.join(savedir, 'training_state.pkl.zip'))
  if container is not None:
    logger.log("Attempting to download model from Azure")
    found_model = container.get(savedir, 'training_state.pkl.zip')
  else:
    found_model = os.path.exists(state_path)
  if found_model:
    state = pickle_load(state_path, compression=True)
    model_dir = "model-{}".format(state["num_iters"])
    if container is not None:
      container.get(savedir, model_dir)
    U.load_state(os.path.join(savedir, model_dir, "saved"))
    logger.log("Loaded models checkpoint at {} iterations".format(
        state["num_iters"]))
    return state
521059
def _initialize(self):
        
        meta = getattr(self, ModelBase._meta_attr)
        # read modelfile, convert JSON and load/update model
        if self.param_file is not None:
            self._load()
        LOGGER.debug('model:\n%r', self.model)
        # initialize layers
        # FIXME: move import inside loop for custom layers in different modules
        mod = importlib.import_module(meta.layers_mod, meta.layers_pkg)
        src_model = {}
        for layer, value in self.model.iteritems():
            # from layers module get the layer's class definition
            layer_cls = getattr(mod, meta.layer_cls_names[layer])  # class def
            self.layers[layer] = layer_cls  # add layer class def to model
            # check if model layers are classes
            src_value = {}  # layer value generated from source classes
            for src in value['sources']:
                # check if source has keyword arguments
                try:
                    src, kwargs = src
                except (TypeError, ValueError):
                    kwargs = {}  # no key work arguments
                # skip if not a source class
                if isinstance(src, basestring):
                    continue
                # generate layer value from source class
                src_value[src.__name__] = {'module': src.__module__,
                                           'package': None}
                # update layer keyword arguments
                src_value[src.__name__].update(kwargs)
            # use layer values generated from source class
            if src_value:
                value = src_model[layer] = src_value
            else:
                srcmod, srcpkg = value.get('module'), value.get('package')
                try:
                    value = dict(value['sources'])
                except ValueError:
                    value = dict.fromkeys(value['sources'], {})
                for src in value.viewkeys():
                    if srcmod is not None:
                        value[src]['module'] = srcmod
                    if srcpkg is not None:
                        value[src]['package'] = srcpkg
            # set layer attribute with model data
            setattr(self, layer, layer_cls(value))
        # update model with layer values generated from source classes
        if src_model:
            self.model.update(src_model)
        self._update()
        self._state = 'initialized'
512743
def _load_models(self) -> None:
        """"""
        if self._models is None:
            logging.info('Loading %d models', len(self._model_paths))

            def load_model(model_path: str):
                logging.debug('\tloading %s', model_path)
                if path.isdir(model_path):
                    model_path = path.join(model_path, CXF_CONFIG_FILE)
                config = load_config(model_path)
                config['model']['inputs'] = self._inputs
                config['model']['outputs'] = self._outputs

                return create_model(config, output_dir=None, dataset=self._dataset,
                                    restore_from=path.dirname(model_path))

            self._models = list(map(load_model, self._model_paths))
1295197
def startup():
    
    model.setup()
    index.scan_index(config.content_folder)
    index.background_scan(config.content_folder)
459160
def run(self):
        
        device = self.model_config.torch_device()
        learner = Learner(device, self.model_factory.instantiate())

        # All callbacks useful for learning
        callbacks = self.gather_callbacks()

        # Metrics to track through this training
        metrics = learner.metrics()

        # Check if training was already started and potentially continue where we left off
        training_info, hidden_state = self.resume_training(learner, callbacks, metrics)

        # Prepare current training phase
        current_phase_idx = self._select_phase_left_bound(training_info.start_epoch_idx)
        current_phase = self.phases[current_phase_idx]
        local_idx = training_info.start_epoch_idx - self.ladder[current_phase_idx]

        current_phase.set_up_phase(training_info, learner.model, self.source)
        print(current_phase.banner())

        if training_info.start_epoch_idx > 0:
            current_phase.restore(training_info, local_idx, learner.model, hidden_state)

        training_info.on_train_begin()

        for global_epoch_idx in range(training_info.start_epoch_idx + 1, self.full_number_of_epochs + 1):
            iteration_phase_idx = self._select_phase_right_bound(global_epoch_idx-1)
            local_idx = global_epoch_idx - self.ladder[iteration_phase_idx]

            # Phase preparations
            while current_phase_idx != iteration_phase_idx:
                current_phase.tear_down_phase(training_info, learner.model)

                current_phase_idx += 1
                current_phase = self.phases[current_phase_idx]

                current_phase.set_up_phase(training_info, learner.model, self.source)
                print(current_phase.banner())

            # Create epoch info
            epoch_info = current_phase.epoch_info(training_info, global_epoch_idx, local_idx)

            # Execute learning
            current_phase.execute_epoch(epoch_info, learner)

            # Epoch checkpoint
            self.storage.checkpoint(epoch_info, learner.model)

        # Tear down the last phase
        if current_phase is not None:
            current_phase.tear_down_phase(training_info, learner.model)

        training_info.on_train_end()

        return training_info
815009
def train(net, train_data, test_data):
    """"""
    start_pipeline_time = time.time()
    net, trainer = text_cnn.init(net, vocab, args.model_mode, context, args.lr)
    random.shuffle(train_data)
    sp = int(len(train_data)*0.9)
    train_dataloader = DataLoader(dataset=train_data[:sp],
                                  batch_size=args.batch_size,
                                  shuffle=True)
    val_dataloader = DataLoader(dataset=train_data[sp:],
                                batch_size=args.batch_size,
                                shuffle=False)
    test_dataloader = DataLoader(dataset=test_data,
                                 batch_size=args.batch_size,
                                 shuffle=False)
    # Training/Testing.
    best_val_acc = 0
    for epoch in range(args.epochs):
        # Epoch training stats.
        start_epoch_time = time.time()
        epoch_L = 0.0
        epoch_sent_num = 0
        epoch_wc = 0
        # Log interval training stats.
        start_log_interval_time = time.time()
        log_interval_wc = 0
        log_interval_sent_num = 0
        log_interval_L = 0.0
        for i, (data, label) in enumerate(train_dataloader):
            data = mx.nd.transpose(data.as_in_context(context))
            label = label.as_in_context(context)
            wc = max_len
            log_interval_wc += wc
            epoch_wc += wc
            log_interval_sent_num += data.shape[1]
            epoch_sent_num += data.shape[1]

            with autograd.record():
                output = net(data)
                L = loss(output, label).mean()
            L.backward()
            # Update parameter.
            trainer.step(1)
            log_interval_L += L.asscalar()
            epoch_L += L.asscalar()
            if (i + 1) % args.log_interval == 0:
                print('[Epoch %d Batch %d/%d] avg loss %g, throughput %gK wps' % (
                    epoch, i + 1, len(train_dataloader),
                    log_interval_L / log_interval_sent_num,
                    log_interval_wc / 1000 / (time.time() - start_log_interval_time)))
                # Clear log interval training stats.
                start_log_interval_time = time.time()
                log_interval_wc = 0
                log_interval_sent_num = 0
                log_interval_L = 0
        end_epoch_time = time.time()
        val_avg_L, val_acc = evaluate(net, val_dataloader)
        print('[Epoch %d] train avg loss %g, '
              'test acc %.4f, test avg loss %g, throughput %gK wps' % (
                  epoch, epoch_L / epoch_sent_num,
                  val_acc, val_avg_L,
                  epoch_wc / 1000 / (end_epoch_time - start_epoch_time)))

        if val_acc >= best_val_acc:
            print('Observed Improvement.')
            best_val_acc = val_acc
            test_avg_L, test_acc = evaluate(net, test_dataloader)

    print('Test loss %g, test acc %.4f'%(test_avg_L, test_acc))
    print('Total time cost %.2fs'%(time.time()-start_pipeline_time))
    return test_acc
611975
def load(self, file_parser, section_name):
        
        self._load(file_parser, section_name)
        self.post_load()