Check for unused and missing sources.

        :param batch: batch to be checked
        :raise ValueError: if a source is missing or unused and ``self._on_unused_sources`` is set to ``error``
196133
def _check_sources(self, batch: Dict[str, object]) -> None:
        
        unused_sources = [source for source in batch.keys() if source not in self._model.input_names]
        missing_sources = [source for source in self._model.input_names if source not in batch.keys()]
        # check stream sources
        if unused_sources:
            if self._on_unused_sources == 'warn' and not self._extra_sources_warned:
                logging.warning('Some sources provided by the stream do not match model placeholders. Set '
                                '`main_loop.on_unused_sources` to `ignore` in order to suppress this warning. '
                                'Extra sources: %s', unused_sources)
                self._extra_sources_warned = True
            elif self._on_unused_sources == 'error':
                raise ValueError('Some sources provided by the stream do not match model placeholders. Set'
                                 '`main_loop.on_unused_sources` to `warn` in order to suppress this error.\n'
                                 'Extra sources: {}'.format(unused_sources))

        if missing_sources:
            raise ValueError('Stream does not provide all required sources. Missing sources: {}'
                             .format(missing_sources))
569789
private void doBatchEnd(final JsonObject message) {
    if (currentBatch != null) {
      Object args = deserializer.deserialize(message);
      if (log.isDebugEnabled()) {
        log.debug(String.format("%s - Batch ended: Batch[batch=%s, args=%s]", this, currentBatch.id(), args));
      }
      currentBatch.handleEnd(args);
      currentBatch = null;
    }
  }
751945
def _find_batch_containing_event(self, uuid):
        
        if self.estore.key_exists(uuid):
            # Reusing already opened DB if possible
            return self.batchno
        else:
            for batchno in range(self.batchno - 1, -1, -1):
                # Iterating backwards here because we are more likely to find
                # the event in an later archive, than earlier.
                db = self._open_event_store(batchno)
                with contextlib.closing(db):
                    if db.key_exists(uuid):
                        return batchno
        return None
192146
protected void checkForBatchSupport(Connection conn)
    {
        if (!m_batchUpdatesChecked)
        {
            DatabaseMetaData meta;
            try
            {
                meta = conn.getMetaData();
                m_supportsBatchUpdates = meta.supportsBatchUpdates();
            }
            catch (Throwable th)
            {
                log.info("Batch support check failed", th);
                m_supportsBatchUpdates = false;
            }
            finally
            {
                m_batchUpdatesChecked = true;
            }
        }
    }
1215787
def check_input_stream_count(expected_number_of_streams):
    

    def stream_count_decorator(func):
        def func_wrapper(*args, **kwargs):
            self = args[0]
            sources = kwargs['sources'] if 'sources' in kwargs else args[1]
            if expected_number_of_streams == 0:
                if sources:
                    raise ValueError("No input streams expected")
            else:
                given_number_of_streams = len(sources) if sources else 0
                if given_number_of_streams != expected_number_of_streams:
                    raise ValueError("{} tool takes {} stream(s) as input ({} given)".format(
                        self.__class__.__name__, expected_number_of_streams, given_number_of_streams))
            return func(*args, **kwargs)

        return func_wrapper

    return stream_count_decorator
580435
def check_missing(self, param, action):
        
        assert action in ('debug', 'info', 'warn', 'error'), action
        if self.inputs.get(param):
            msg = '%s_file in %s is ignored in %s' % (
                param, self.inputs['job_ini'], self.calculation_mode)
            if action == 'error':
                raise InvalidFile(msg)
            else:
                getattr(logging, action)(msg)
570454
def on_batch_end(self, train, **kwargs):
        ""
        if train: self.stats.append(self.hooks.stored)
659292
def _check_cat_dict_source(self, cat_dict_class, key_in_self, **kwargs):
        """"""
        # Make sure that a source is given
        source = kwargs.get(cat_dict_class._KEYS.SOURCE, None)
        if source is None:
            raise CatDictError(
                "{}: `source` must be provided!".format(self[self._KEYS.NAME]),
                warn=True)
        # Check that source is a list of integers
        for x in source.split(','):
            if not is_integer(x):
                raise CatDictError(
                    "{}: `source` is comma-delimited list of "
                    " integers!".format(self[self._KEYS.NAME]),
                    warn=True)
        # If this source/data is erroneous, skip it
        if self.is_erroneous(key_in_self, source):
            self._log.info("This source is erroneous, skipping")
            return None
        # If this source/data is private, skip it
        if (self.catalog.args is not None and not self.catalog.args.private and
                self.is_private(key_in_self, source)):
            self._log.info("This source is private, skipping")
            return None
        return source
1352410
public void updateBatch(V batch) {

        if (intervalCheck.check()) {
            rolling();
        }
        synchronized (this) {
            unflushed = updater.updateBatch(batch, unflushed);
        }

    }
1287288
def _check_for_batch_clashes(xs):
    
    names = set([x["description"] for x in xs])
    dups = set([])
    for x in xs:
        batches = tz.get_in(("metadata", "batch"), x)
        if batches:
            if not isinstance(batches, (list, tuple)):
                batches = [batches]
            for batch in batches:
                if batch in names:
                    dups.add(batch)
    if len(dups) > 0:
        raise ValueError("Batch names must be unique from sample descriptions.\n"
                         "Clashing batch names: %s" % sorted(list(dups)))
655309
def available_args(self):
        
        used = list(self.used_args)
        logger.debug('Found used args: %s' % used)
        for arg in list(self.cmd.args.values()):
            if (arg.is_multiple or
                    arg not in used):
                yield arg
            elif (type(arg.nargs) is int and
                    arg.nargs > 1 and
                    not arg.nargs == used.count(arg)):
                yield arg
665798
def files_exist(self):
        
        if not os.path.isfile(self.path + self.script):
            print("\nslpkg: Error: SlackBuild archive.tar.gz not found\n")
            raise SystemExit()
        for src in self.sources:
            if not os.path.isfile(self.path + src):
                print("\nslpkg: Error: Source file '{0}' not found\n".format(
                    src))
                raise SystemExit()
102111
def _loadBatch(self, item, batch):
        
        if self.isThreadEnabled() and batch.isThreadEnabled():
            self.loadBatchRequested.emit(batch)
            self._batchloaders.append(weakref.ref(item))
        else:
            QApplication.setOverrideCursor(Qt.WaitCursor)
            self.worker().loadRecords(batch)
            QApplication.restoreOverrideCursor()
1105340
void onSealExclusive(Batch batch)
  {
    buffersToEmit.add(batch);
    wakeUpEmittingThread();
    if (!isTerminated()) {
      int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(batch.batchNumber);
      if (!concurrentBatch.compareAndSet(batch, new Batch(this, acquireBuffer(), nextBatchNumber))) {
        // If compareAndSet failed, the service is closed concurrently.
        Preconditions.checkState(isTerminated());
      }
    }
  }
1024901
function removeEmptySources (sources) {
    return _.reject(sources, function (obj) {
        return _.isEmpty(obj.transformed);
    });
}
225527
def sanity_check(self):
        ""
        final_message = "You can deactivate this warning by passing `no_check=True`."
        if not hasattr(self.train_ds, 'items') or len(self.train_ds.items) == 0 or not hasattr(self.train_dl, 'batch_sampler'): return
        if len(self.train_dl) == 0:
            warn(f"""Your training dataloader is empty, you have only {len(self.train_dl.dataset)} items in your training set.
                 Your batch size is {self.train_dl.batch_size}, you should lower it.""")
            print(final_message)
            return
        idx = next(iter(self.train_dl.batch_sampler))
        samples,fails = [],[]
        for i in idx:
            try:    samples.append(self.train_dl.dataset[i])
            except: fails.append(i)
        if len(fails) > 0:
            warn_msg = "There seems to be something wrong with your dataset, for example, in the first batch can't access"
            if len(fails) == len(idx):
                warn_msg += f" any element of self.train_ds.\nTried: {show_some(idx)}"
            else:
                warn_msg += f" these elements in self.train_ds: {show_some(fails)}"
            warn(warn_msg)
            print(final_message)
            return
        try: batch = self.collate_fn(samples)
        except:
            message = "It's not possible to collate samples of your dataset together in a batch."
            try:
                shapes = [[o[i].data.shape for o in samples] for i in range(2)]
                message += f'\nShapes of the inputs/targets:\n{shapes}'
            except: pass
            warn(message)
            print(final_message)
353202
def _check_existing_logger(loggername, short_name):
    
    if loggername in LOGGERS:
        # Check if short_name matches the existing one, if not update it
        if isinstance(LOGGERS[loggername], BenchLoggerAdapter):
            if ("source" not in LOGGERS[loggername].extra or
                    LOGGERS[loggername].extra["source"] != short_name):
                LOGGERS[loggername].extra["source"] = short_name
        return LOGGERS[loggername]
    return None