Gather all the callbacks to be used in this training run
498223
def gather_callbacks(self) -> list:
        
        callbacks = []

        callbacks.extend(self.callbacks)
        callbacks.extend(self.storage.streaming_callbacks())

        return callbacks
705290
def train_all(ctx, output):
    """"""
    click.echo('chemdataextractor.pos.train_all')
    click.echo('Output: %s' % output)
    ctx.invoke(train, output='%s_wsj_nocluster.pickle' % output, corpus='wsj', clusters=False)
    ctx.invoke(train, output='%s_wsj.pickle' % output, corpus='wsj', clusters=True)
    ctx.invoke(train, output='%s_genia_nocluster.pickle' % output, corpus='genia', clusters=False)
    ctx.invoke(train, output='%s_genia.pickle' % output, corpus='genia', clusters=True)
    ctx.invoke(train, output='%s_wsj_genia_nocluster.pickle' % output, corpus='wsj+genia', clusters=False)
    ctx.invoke(train, output='%s_wsj_genia.pickle' % output, corpus='wsj+genia', clusters=True)
939554
def _run_progress_callbacks(self, bytes_transferred):
        '''  '''
        if bytes_transferred:
            for callback in self._progress_callbacks:
                try:
                    callback(bytes_transferred=bytes_transferred)
                except Exception as ex:
                    logger.error("Exception: %s" % str(ex))
1212734
def trigger_all_change_callbacks(self):
        """"""
        return [
            ret
            for key in DatastoreLegacy.store[self.domain].keys()
            for ret in self.trigger_change_callbacks(key)
        ]
623373
def process_callbacks(event_type, event_data)
      @callbacks[event_type].each do |callback|
        fiber = Fiber.new do
          begin
            callback.call(event_data)
          rescue => e
            @loggers.error "-- #{e.class}: #{e.message}"
            e.backtrace.each { |line| @loggers.error("-- #{line}") }
          end
        end

        # If the callback has a :defer option, call it in a thread
        # from the EM thread pool. Else call it in the reactor thread.
        if callback.options[:defer]
          EM.defer(fiber.resume)
        else
          fiber.resume
        end
      end
    end
286684
def run(self):
        
        device = self.model_config.torch_device()

        learner = api.Learner(device, self.model_factory.instantiate(), self.max_grad_norm)
        optimizer = self.optimizer_factory.instantiate(learner.model)

        # All callbacks used for learning
        callbacks = self.gather_callbacks(optimizer)

        # Metrics to track through this training
        metrics = learner.metrics()

        # Check if training was already started and potentially continue where we left off
        training_info = self.resume_training(learner, callbacks, metrics)

        training_info.on_train_begin()

        if training_info.optimizer_initial_state:
            optimizer.load_state_dict(training_info.optimizer_initial_state)

        for global_epoch_idx in range(training_info.start_epoch_idx + 1, self.epochs + 1):
            epoch_info = api.EpochInfo(
                training_info=training_info,
                global_epoch_idx=global_epoch_idx,
                batches_per_epoch=self.source.train_iterations_per_epoch(),
                optimizer=optimizer
            )

            # Execute learning
            learner.run_epoch(epoch_info, self.source)

            self.storage.checkpoint(epoch_info, learner.model)

        training_info.on_train_end()

        return training_info
53973
def retrain(self):
        """"""
        folder = TrainData.from_folder(self.args.folder)
        train_data, test_data = folder.load(True, not self.args.no_validation)

        train_data = TrainData.merge(train_data, self.sampled_data)
        test_data = TrainData.merge(test_data, self.test)
        train_inputs, train_outputs = train_data
        print()
        try:
            self.listener.runner.model.fit(
                train_inputs, train_outputs, self.args.batch_size, self.epoch + self.args.epochs,
                validation_data=test_data, callbacks=self.callbacks, initial_epoch=self.epoch
            )
        finally:
            self.listener.runner.model.save(self.args.model)
396462
def handlers(self):
        """"""
        if getattr(self, '_http'):
            yield from self.http.handlers()
        if getattr(self, '_cli'):
            yield from self.cli.handlers()
700436
def setup_callbacks(self, callbacks, monitors):
        
        assert isinstance(callbacks, list), callbacks
        assert isinstance(monitors, list), monitors
        describe_trainable_vars()   # TODO weird

        self.register_callback(MaintainStepCounter())
        for cb in callbacks:
            self.register_callback(cb)
        for cb in self._callbacks:
            assert not isinstance(cb, MonitorBase), "Monitor cannot be pre-registered for now!"
        registered_monitors = []
        for m in monitors:
            if self.register_callback(m):
                registered_monitors.append(m)
        self.monitors = Monitors(registered_monitors)
        self.register_callback(self.monitors)   # monitors is also a callback

        # some final operations that might modify the graph
        logger.info("Setup callbacks graph ...")
        self._callbacks = Callbacks(self._callbacks)
        self._callbacks.setup_graph(weakref.proxy(self))
278518
def process_events(self):
        """"""
        for event, args in iter_except(self.event_queue.popleft, IndexError):
            for callback in self.event_callbacks[event]:
                callback(*args)
897264
def _try_run(self, run_func: Callable[[], None]) -> None:
        
        # Initialization: before_training
        for hook in self._hooks:
            hook.before_training()

        try:
            run_func()
        except TrainingTerminated as ex:
            logging.info('Training terminated: %s', ex)

        # After training: after_training
        for hook in self._hooks:
            hook.after_training()
395737
def run_callbacks_for(name, args, reverse)
      callbacks_for(name, reverse).map do |block|
        run_callback(block, args)
      end
    end
1207244
def _invoke_callbacks(self, *args, **kwargs):
        """"""
        for callback in self._done_callbacks:
            _helpers.safe_invoke_callback(callback, *args, **kwargs)
1290843
def _call_callbacks(self):
        """"""
        for callback in self._callbacks:
            callback(self, self._changeset)
        self._changeset = {}
1034124
def _run_atstart():
    ''''''
    global _atstart
    for callback, args, kwargs in _atstart:
        callback(*args, **kwargs)
    del _atstart[:]
601504
def _event_funcs(self, event: str) -> Iterable[Callable]:
        
        for func in self._events[event]:
            yield func
446538
def on_step_end(self, step, logs={}):
        
        for callback in self.callbacks:
            # Check if callback supports the more appropriate `on_step_end` callback.
            # If not, fall back to `on_batch_end` to be compatible with built-in Keras callbacks.
            if callable(getattr(callback, 'on_step_end', None)):
                callback.on_step_end(step, logs=logs)
            else:
                callback.on_batch_end(step, logs=logs)