Get learning rate with cosine decay based on current iteration.

        Args:
            iter (int): Current iteration (starting with 0).

        Returns:
            float: Learning rate
334686
def get_learning_rate(self, iter):
        '''
        
        '''
        return self.init_lr * ((math.cos(iter * 1.0 / (self.max_iter) * math.pi) + 1.0) * 0.5)
277576
def _warmup(self, num_updates):
        
        assert self.base_lr is not None
        if not self.warmup:
            return self.base_lr
        fraction = (num_updates + 1) * self.base_lr / (self.warmup + 1)
        if num_updates > self.last_warmup_log and num_updates % self.log_warmup_every_t == 0:
            self.last_warmup_log = num_updates
            logger.info("Learning rate warmup: %3.0f%%", fraction / self.base_lr * 100.0)
        return fraction
63400
def iter(self):
        
        # Note that self._reader is already initialized (in __init__) for
        # the first input file. This is less clean than it could be, but it
        # makes testing easier, since open() is then only called once for
        # each input file.

        reads = iter(self.reads)
        first = True

        for filename in self.filenames:
            if first:
                # The first file has already been opened, in __init__.
                first = False
                reader = self._reader
            else:
                reader = self._getReader(filename, self.scoreClass)

            for readAlignments in reader.readAlignments(reads):
                yield readAlignments

        # Any remaining query reads must have had no subject matches.
        for read in reads:
            yield ReadAlignments(read, [])
648967
def feed_data(self, data_loader, training=True):
        
        if training:
            assert self.optimizer is not None
            eval_fractions = np.linspace(0, 1, self.intra_epoch_eval+2)[1:-1]
            iters_with_update = len(data_loader) // self.iter_size
            eval_iters = (eval_fractions * iters_with_update).astype(int)
            eval_iters = eval_iters * self.iter_size
            eval_iters = set(eval_iters)

        batch_time = AverageMeter()
        data_time = AverageMeter()
        losses_per_token = AverageMeter(skip_first=False)
        losses_per_sentence = AverageMeter(skip_first=False)

        tot_tok_time = AverageMeter()
        src_tok_time = AverageMeter()
        tgt_tok_time = AverageMeter()

        batch_size = data_loader.batch_size

        end = time.time()
        for i, (src, tgt) in enumerate(data_loader):
            self.save_counter += 1
            # measure data loading time
            data_time.update(time.time() - end)

            update = False
            if i % self.iter_size == self.iter_size - 1:
                update = True

            # do a train/evaluate iteration
            stats = self.iterate(src, tgt, update, training=training)
            loss_per_token, loss_per_sentence, num_toks = stats

            # measure accuracy and record loss
            losses_per_token.update(loss_per_token, num_toks['tgt'])
            losses_per_sentence.update(loss_per_sentence, batch_size)

            # measure elapsed time
            elapsed = time.time() - end
            batch_time.update(elapsed)
            src_tok_time.update(num_toks['src'] / elapsed)
            tgt_tok_time.update(num_toks['tgt'] / elapsed)
            tot_num_toks = num_toks['tgt'] + num_toks['src']
            tot_tok_time.update(tot_num_toks / elapsed)
            self.loss = losses_per_token.avg

            if training and i in eval_iters:
                test_bleu, _ = self.translator.run(calc_bleu=True,
                                                   epoch=self.epoch,
                                                   iteration=i)

                log = []
                log += [f'TRAIN [{self.epoch}][{i}/{len(data_loader)}]']
                log += [f'BLEU: {test_bleu:.2f}']
                log = '\t'.join(log)
                logging.info(log)

                self.model.train()
                self.preallocate(data_loader, training=True)

            if i % self.print_freq == 0:
                phase = 'TRAIN' if training else 'VALIDATION'
                log = []
                log += [f'{phase} [{self.epoch}][{i}/{len(data_loader)}]']
                log += [f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})']
                log += [f'Data {data_time.val:.2e} ({data_time.avg:.2e})']
                log += [f'Tok/s {tot_tok_time.val:.0f} ({tot_tok_time.avg:.0f})']
                if self.verbose:
                    log += [f'Src tok/s {src_tok_time.val:.0f} ({src_tok_time.avg:.0f})']
                    log += [f'Tgt tok/s {tgt_tok_time.val:.0f} ({tgt_tok_time.avg:.0f})']
                    log += [f'Loss/sentence {losses_per_sentence.val:.1f} ({losses_per_sentence.avg:.1f})']
                log += [f'Loss/tok {losses_per_token.val:.4f} ({losses_per_token.avg:.4f})']
                if training:
                    lr = self.optimizer.param_groups[0]['lr']
                    log += [f'LR {lr:.3e}']
                log = '\t'.join(log)
                logging.info(log)

            save_chkpt = (self.save_counter % self.save_freq) == (self.save_freq - 1)
            if training and save_chkpt:
                self.save_counter = 0
                self.save_info['iteration'] = i
                identifier = next(self.checkpoint_counter, -1)
                if identifier != -1:
                    with sync_workers() as rank:
                        if rank == 0:
                            self.save(identifier=identifier)

            end = time.time()

        tot_tok_time.reduce('sum')
        losses_per_token.reduce('mean')

        return losses_per_token.avg, tot_tok_time.avg
840525
def get_mnist_iter(args, kv):
    
    (train_lbl, train_img) = read_data(
            'train-labels-idx1-ubyte.gz', 'train-images-idx3-ubyte.gz')
    (val_lbl, val_img) = read_data(
            't10k-labels-idx1-ubyte.gz', 't10k-images-idx3-ubyte.gz')
    train = mx.io.NDArrayIter(
        to4d(train_img), train_lbl, args.batch_size, shuffle=True)
    val = mx.io.NDArrayIter(
        to4d(val_img), val_lbl, args.batch_size)
    return (train, val)
1144889
def get_iterator(self, data, axis=0):
        
        if isinstance(data, NDFrame):
            slicer = lambda start, edge: data._slice(
                slice(start, edge), axis=axis)
            length = len(data.axes[axis])
        else:
            slicer = lambda start, edge: data[slice(start, edge)]
            length = len(data)

        start = 0
        for edge, label in zip(self.bins, self.binlabels):
            if label is not NaT:
                yield label, slicer(start, edge)
            start = edge

        if start < length:
            yield self.binlabels[-1], slicer(start, None)
5030
def iter_rate(self):
        
        #class IterState(object):
        #    def __init__(state):
        #        state.freq = 1
        #        state.freq = 1
        #        pass
        adjust = self.autoadjust
        self._cursor_at_newline = not self.backspace
        # SETUP VARIABLES
        # HACK: reaquire logging print funcs in case they have changed
        if self.stream is None:
            self.write = util_logging._utool_write()
            self.flush = util_logging._utool_flush()
        else:
            self.write = lambda msg: self.stream.write(msg)  # NOQA
            self.flush = lambda: self.stream.flush()  # NOQA

        length        = self.length * self.parent_length  # hack
        freq          = self.freq
        self.count    = 0
        between_count = 0
        last_count    = 0

        # how long iterations should be before a flush
        # (used for freq adjustment)
        time_thresh = (self._get_timethresh_heuristics()
                       if self.time_thresh is None else
                       self.time_thresh)
        time_thresh_growth = self.time_thresh_growth
        if time_thresh_growth > 1:
            # time_thresh_growth is specified for very long processes
            # print out the starting timestamp in that case
            timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + time.tzname[0]
            print('Start progress lbl= %s at %s' % (self.lbl, timestamp,))
        #time_thresh = 0.5
        max_between_time = -1.0
        max_between_count = -1.0  # why is this different? # because frequency varies

        # TODO: should be kept as a statistic that uses the max time from a
        # list of iterations divided by the size of that list that will account
        # for buffering issues
        iters_per_second = 0
        self.iters_per_second = float('nan')
        self.est_seconds_left = 0
        self.total_seconds = 0

        # Write initial message
        #force_newlines = not self.backspace
        start_msg_fmt = ''.join(self.build_msg_fmtstr_head_cols(length, self.lbl))
        self.msg_fmtstr = self.build_msg_fmtstr2(self.lbl, length,
                                                 self.invert_rate,
                                                 self.backspace)

        try:
            util_logging._utool_flush()()
        except IOError as ex:
            # There is some weird error when doing progress in IPython notebook
            if util_arg.VERBOSE:
                print('IOError flushing %s' % (ex,))
        if not self.prehack:
            if self.backspace:
                self.display_message()
            elif self.verbose:
                start_msg = start_msg_fmt.format(count=self.parent_offset)
                util_logging._utool_write()(start_msg + '\n')

            self._cursor_at_newline = not self.backspace

            try:
                util_logging._utool_flush()()
            except IOError as ex:
                # There is some weird error when doing progress in IPython notebook
                if util_arg.VERBOSE:
                    print('IOError flushing %s' % (ex,))
        else:
            self._cursor_at_newline = True

        if self.prog_hook is not None:
            self.prog_hook(self.count, length)

        # TODO: on windows is time.clock better?
        # http://exnumerus.blogspot.com/2011/02/how-to-quickly-plot-multiple-line.html
        start_time    = default_timer()
        last_time     = start_time

        start = 1 + self.parent_offset

        if self.freq_est_strat == 'between':
            FREQ_EST = 0
        elif self.freq_est_strat == 'absolute':
            FREQ_EST = 1
        else:
            FREQ_EST = 1

        USE_RECORD = True

        # use last 64 times to compute a more stable average rate
        measure_between_time = collections.deque([], maxlen=self.est_window)

        # Wrap the for loop with a generator
        for self.count, item in enumerate(self.iterable, start=start):
            if self.prehack:
                # hack to print before yeilding
                # so much for efficiency
                self.set_extra((self.lbl + '=' + self.prehack) % item)
                self.display_message()
                self.ensure_newline()

            # GENERATE
            yield item

            if self.prehack or (self.count) % freq == 0:
                now_time          = default_timer()
                between_time      = (now_time - last_time)
                between_count     = self.count - last_count
                total_seconds     = (now_time - start_time)
                self.total_seconds = total_seconds
                if FREQ_EST == 0:
                    if USE_RECORD:
                        measure_between_time.append(between_count / (float(between_time) + 1E-9))
                        iters_per_second = sum(measure_between_time) / len(measure_between_time)
                    else:
                        iters_per_second = between_count / (float(between_time) + 1E-9)
                elif FREQ_EST == 1:
                    iters_per_second = (now_time - start_time) / self.count

                self.iters_per_second = iters_per_second
                # If the future is known
                if length is None:
                    est_seconds_left = -1
                else:
                    iters_left = length - self.count
                    est_seconds_left = iters_left / (iters_per_second + 1E-9)
                self.est_seconds_left = est_seconds_left

                # /future
                last_count        = self.count
                last_time         = now_time
                # ADJUST FREQ IF NEEDED
                # Adjust frequency if printing too quickly
                # so progress doesnt slow down actual function
                # TODO: better adjust algorithm
                time_thresh *= time_thresh_growth
                if adjust and (between_time < time_thresh or between_time > time_thresh * 2.0):
                    max_between_time = max(max(max_between_time, between_time),
                                           1E-9)
                    max_between_count = max(max_between_count, between_count)
                    # If progress was uniform and all time estimates were
                    # perfect this would be the new freq to achieve time_thresh
                    new_freq = max(int(time_thresh * max_between_count /
                                       max_between_time), 1)
                    if DEBUG_FREQ_ADJUST:
                        print('\n+---')
                        print('[prog] between_count = %r' % between_count)
                        print('[prog] between_time = %.8r' % between_time)
                        print('[prog] time_thresh = %r' % time_thresh)
                        print('[prog] max_between_count = %r' % max_between_count)
                        print('[prog] max_between_time = %.8r' % max_between_time)
                        print('[prog] Adusting frequency from: %r' % freq)
                        print('[prog] Adusting frequency to: %r' % new_freq)
                        print('L___')
                    # But things are not perfect. So, don't make drastic changes
                    max_freq_change_up = max(256, freq * 2)
                    max_freq_change_down = freq // 2
                    if (new_freq - freq) > max_freq_change_up:
                        freq += max_freq_change_up
                    elif (freq - new_freq) > max_freq_change_down:
                        freq -= max_freq_change_down
                    else:
                        freq = new_freq

                if not self.prehack:
                    self.display_message()

                # DO PROGRESS INFO
                if self.prog_hook is not None:
                    # From the point of view of the progress iter, we are about
                    # to enter the body of a for loop. (But we may have
                    # executed the body implicitly in the yeild....  so it is
                    # ambiguous. In the second case 0 will be executed twice.
                    self.prog_hook(self.count, length)

        if self.prehack:
            self.set_extra('')

        # --- end of main loop
        # cleanup
        if (self.count) % freq != 0:
            # If the final line of progress was not written in the loop, write
            # it here
            self.est_seconds_left = 0
            self.total_seconds = (default_timer() - start_time)
            self.display_message()
            if self.prog_hook is not None:
                # From the point of view of the progress iter, we are about to
                # enter the body of a for loop. (But we may have executed the
                # body implicitly in the yeild....  so it is ambiguous. In the
                # second case 0 will be executed twice.
                self.prog_hook(self.count, length)

        self.ensure_newline()
1247754
def current_iteration(self):
        
        out_cur_iter = ctypes.c_int(0)
        _safe_call(_LIB.LGBM_BoosterGetCurrentIteration(
            self.handle,
            ctypes.byref(out_cur_iter)))
        return out_cur_iter.value
1105849
def _load_training_state(self, train_iter: data_io.BaseParallelSampleIter):
        
        # (1) Parameters
        params_fname = os.path.join(self.training_state_dirname, C.TRAINING_STATE_PARAMS_NAME)
        self.model.load_params_from_file(params_fname)

        # (2) Optimizer states
        opt_state_fname = os.path.join(self.training_state_dirname, C.OPT_STATES_LAST)
        self.model.load_optimizer_states(opt_state_fname)

        # (3) Data Iterator
        train_iter.load_state(os.path.join(self.training_state_dirname, C.BUCKET_ITER_STATE_NAME))

        # (4) Random generators
        # RNG states: python's random and np.random provide functions for
        # storing the state, mxnet does not, but inside our code mxnet's RNG is
        # not used AFAIK
        with open(os.path.join(self.training_state_dirname, C.RNG_STATE_NAME), "rb") as fp:
            random.setstate(pickle.load(fp))
            np.random.set_state(pickle.load(fp))

        # (5) Training state
        self.state = TrainState.load(os.path.join(self.training_state_dirname, C.TRAINING_STATE_NAME))

        # (6) Learning rate scheduler
        with open(os.path.join(self.training_state_dirname, C.SCHEDULER_STATE_NAME), "rb") as fp:
            self.optimizer_config.set_lr_scheduler(pickle.load(fp))
        # initialize optimizer again
        self._initialize_optimizer()
77566
def train_epoch(self, epoch_info: EpochInfo, interactive=True):
        
        epoch_info.on_epoch_begin()

        if interactive:
            iterator = tqdm.trange(epoch_info.batches_per_epoch, file=sys.stdout, desc="Training", unit="batch")
        else:
            iterator = range(epoch_info.batches_per_epoch)

        for batch_idx in iterator:
            batch_info = BatchInfo(epoch_info, batch_idx)

            batch_info.on_batch_begin()
            self.train_batch(batch_info)
            batch_info.on_batch_end()

        epoch_info.result_accumulator.freeze_results()
        epoch_info.on_epoch_end()
852332
def get_lr(self, step, nowarn=False):
        
        if self.t_total < 0:
            return 1.
        progress = float(step) / self.t_total
        ret = self.get_lr_(progress)
        # warning for exceeding t_total (only active with warmup_linear
        if not nowarn and self.warn_t_total and progress > 1. and progress > self.warned_for_t_total_at_progress:
            logger.warning(
                "Training beyond specified 't_total'. Learning rate multiplier set to {}. Please set 't_total' of {} correctly."
                    .format(ret, self.__class__.__name__))
            self.warned_for_t_total_at_progress = progress
        # end warning
        return ret
1346700
def read_iter():
    '''
    '''
    filename = 'exe/inv.lastmod'
    linestring = open(filename, 'r').readline().strip()
    linestring = linestring.replace('\n', '')
    linestring = linestring.replace('../', '')
    return linestring
376084
def get_epoch_iterator(self, **kwargs):
        """"""
        if not self._fresh_state:
            self.next_epoch()
        else:
            self._fresh_state = False
        return super(DataStream, self).get_epoch_iterator(**kwargs)
217319
def iter_values(self):
        
        yVal = self._element.yVal
        if yVal is None:
            return

        for idx in range(yVal.ptCount_val):
            yield yVal.pt_v(idx)
1255589
def get_learning_rate(self, iter):
        '''
        
        '''
        lr = self.init_lr
        for iter_step in self.iter_steps:
            if iter >= iter_step:
                lr *= self.gamma
        return lr
1291469
def iter(self):
        
        if self._upperCase:
            for id_ in self._fasta:
                yield self._readClass(id_, str(self._fasta[id_]).upper())
        else:
            for id_ in self._fasta:
                yield self._readClass(id_, str(self._fasta[id_]))
521133
def _learning_rate_decay(hparams, warmup_steps=0):
  """"""
  scheme = hparams.learning_rate_decay_scheme
  warmup_steps = tf.to_float(warmup_steps)
  global_step = _global_step(hparams)

  if not scheme or scheme == "none":
    return tf.constant(1.)

  tf.logging.info("Applying learning rate decay: %s.", scheme)

  if scheme == "exp":
    decay_steps = hparams.learning_rate_decay_steps
    p = (global_step - warmup_steps) / decay_steps
    if hparams.learning_rate_decay_staircase:
      p = tf.floor(p)
    return tf.pow(hparams.learning_rate_decay_rate, p)

  if scheme == "piecewise":
    return _piecewise_learning_rate(global_step,
                                    hparams.learning_rate_boundaries,
                                    hparams.learning_rate_multiples)

  if scheme == "cosine":
    cycle_steps = hparams.learning_rate_cosine_cycle_steps
    cycle_position = global_step % (2 * cycle_steps)
    cycle_position = cycle_steps - tf.abs(cycle_steps - cycle_position)
    return 0.5 * (1 + tf.cos(np.pi * cycle_position / cycle_steps))

  if scheme == "cyclelinear10x":
    # Cycle the rate linearly by 10x every warmup_steps, up and down.
    cycle_steps = warmup_steps
    cycle_position = global_step % (2 * cycle_steps)
    cycle_position = tf.to_float(  # Normalize to the interval [-1, 1].
        cycle_position - cycle_steps) / float(cycle_steps)
    cycle_position = 1.0 - tf.abs(cycle_position)  # 0 to 1 and back to 0.
    return (cycle_position + 0.1) * 3.0  # 10x difference each cycle (0.3-3).

  if scheme == "sqrt":
    return _legacy_sqrt_decay(global_step - warmup_steps)

  raise ValueError("Unrecognized learning rate decay scheme: %s" %
                   hparams.learning_rate_decay_scheme)