Small range of hyperparameters.
977866
def vqa_attention_base_range(rhp):
  """"""
  # After starting from base, set intervals for some parameters.
  rhp.set_float("learning_rate", 0.1, 1.0, scale=rhp.LOG_SCALE)
  rhp.set_float("clip_grad_norm", 0.1, 10, scale=rhp.LOG_SCALE)
  rhp.set_discrete("batch_size", [128, 256, 512, 1024])
  rhp.set_float("weight_decay", 0.0, 1e-4)
  rhp.set_categorical("rnn_type", ["lstm", "lstm_layernorm"])
1053406
def super_lm_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.hidden_size = 512
  hparams.moe_hidden_sizes = "512"
  hparams.batch_size = 16384
  hparams.max_length = 0
  # All hyperparameters ending in "dropout" are automatically set to 0.0
  # when not in training mode.
  hparams.layer_prepostprocess_dropout = 0.0
  hparams.symbol_dropout = 0.1
  hparams.add_hparam("attention_dropout", 0.0)
  hparams.label_smoothing = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 8000
  hparams.initializer_gain = 1.0
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.shared_embedding_and_softmax_weights = False
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  # we only want one data shard.
  hparams.no_data_parallelism = True
  # bypass the symbol modality so that we can use model parallelism.
  hparams.bottom = {
      "inputs": modalities.identity_bottom,
      "targets": modalities.identity_bottom,
  }
  hparams.top = {
      "targets": modalities.identity_top,
  }
  hparams.add_hparam("filter_size", 512)
  hparams.add_hparam("mix_fraction", 0.5)
  # attention-related flags
  hparams.add_hparam("multihead_attention_num_heads", 4)
  hparams.add_hparam("multihead_attention_key_channels", 0)
  hparams.add_hparam("multihead_attention_value_channels", 0)
  hparams.add_hparam("pos", "timing")  # timing, none
  hparams.add_hparam(
      "layers", ("n,att,m,d,a," "n,ffn,m,d,a,") * 4 + "n,ffn,d")
  # Number of model shards - each one has separate parameters.
  # Changing this number invalidates checkpoints.
  hparams.add_hparam("num_model_shards", 8)
  hparams.add_hparam("diet_experts", False)
  return hparams
214995
def autoencoder_ordered_text_small():
  """"""
  hparams = autoencoder_ordered_text()
  hparams.bottleneck_bits = 32
  hparams.num_hidden_layers = 3
  hparams.hidden_size = 64
  hparams.max_hidden_size = 512
  hparams.bottleneck_noise = 0.0
  hparams.autoregressive_mode = "conv5"
  hparams.sample_height = 4
  return hparams
470228
def min_tens(tens, rmax=10, nswp=10, verb=True, smooth_fun=None):
    """"""
    if smooth_fun is None:
        smooth_fun = lambda p, lam: (math.pi / 2 - np.arctan(p - lam))
    d = tens.d
    Rx = [[]] * (d + 1)  # Python list for the interfaces
    Rx[0] = np.ones((1, 1))
    Rx[d] = np.ones((1, 1))
    Jy = [np.empty(0, dtype=np.int)] * (d + 1)
    ry = rmax * np.ones(d + 1, dtype=np.int)
    ry[0] = 1
    ry[d] = 1
    n = tens.n
    elements_seen = 0
    phi_left = [np.empty(0)] * (d + 1)
    phi_left[0] = np.array([1])
    phi_right = [np.empty(0)] * (d + 1)
    phi_right[d] = np.array([1])
    cores = tt.tensor.to_list(tens)

    # Fill initial multiindex J randomly.
    grid = [np.reshape(range(n[i]), (n[i], 1)) for i in xrange(d)]
    for i in xrange(d - 1):
        ry[i + 1] = min(ry[i + 1], n[i] * ry[i])
        ind = sorted(np.random.permutation(ry[i] * n[i])[0:ry[i + 1]])
        w1 = mkron(np.ones((n[i], 1), dtype=np.int), Jy[i])
        w2 = mkron(grid[i], np.ones((ry[i], 1), dtype=np.int))
        Jy[i + 1] = np.hstack((w1, w2))
        Jy[i + 1] = reshape(Jy[i + 1], (ry[i] * n[i], -1))
        Jy[i + 1] = Jy[i + 1][ind, :]
        phi_left[i + 1] = np.tensordot(phi_left[i], cores[i], 1)
        phi_left[i + 1] = reshape(phi_left[i + 1], (ry[i] * n[i], -1))
        phi_left[i + 1] = phi_left[i + 1][ind, :]
    swp = 0
    dirn = -1
    i = d - 1
    lm = float('Inf')
    while swp < nswp:
        # Right-to-left sweep
        # The idea: compute the current core; compute the function of it;
        # Shift locally or globally? Local shift would be the first try
        # Compute the current core

        if np.size(Jy[i]) == 0:
            w1 = np.zeros((ry[i] * n[i] * ry[i + 1], 0), dtype=np.int)
        else:
            w1 = mkron(np.ones((n[i] * ry[i + 1], 1), dtype=np.int), Jy[i])
        w2 = mkron(mkron(np.ones((ry[i + 1], 1), dtype=np.int),
                         grid[i]), np.ones((ry[i], 1), dtype=np.int))
        if np.size(Jy[i + 1]) == 0:
            w3 = np.zeros((ry[i] * n[i] * ry[i + 1], 0), dtype=np.int)
        else:
            w3 = mkron(Jy[i + 1], np.ones((ry[i] * n[i], 1), dtype=np.int))
        J = np.hstack((w1, w2, w3))

        phi_right[i] = np.tensordot(cores[i], phi_right[i + 1], 1)
        phi_right[i] = reshape(phi_right[i], (-1, n[i] * ry[i + 1]))

        cry = np.tensordot(
            phi_left[i], np.tensordot(
                cores[i], phi_right[
                    i + 1], 1), 1)
        elements_seen += cry.size
        cry = reshape(cry, (ry[i], n[i], ry[i + 1]))
        min_cur = np.min(cry.flatten("F"))
        ind_cur = np.argmin(cry.flatten("F"))
        if lm > min_cur:
            lm = min_cur
            x_full = J[ind_cur, :]
            val = tens[x_full]
            if verb:
                print('New record:', val, 'Point:', x_full, 'elements seen:', elements_seen)
        cry = smooth_fun(cry, lm)
        if dirn < 0 and i > 0:
            cry = reshape(cry, (ry[i], n[i] * ry[i + 1]))
            cry = cry.T
            #q, r = np.linalg.qr(cry)
            u, s, v = mysvd(cry, full_matrices=False)
            ry[i] = min(ry[i], rmax)
            q = u[:, :ry[i]]
            ind = rect_maxvol(q)[0]  # maxvol(q)
            ry[i] = ind.size
            w1 = mkron(np.ones((ry[i + 1], 1), dtype=np.int), grid[i])
            if np.size(Jy[i + 1]) == 0:
                w2 = np.zeros((n[i] * ry[i + 1], 0), dtype=np.int)
            else:
                w2 = mkron(Jy[i + 1], np.ones((n[i], 1), dtype=np.int))
            Jy[i] = np.hstack((w1, w2))
            Jy[i] = reshape(Jy[i], (n[i] * ry[i + 1], -1))
            Jy[i] = Jy[i][ind, :]
            phi_right[i] = np.tensordot(cores[i], phi_right[i + 1], 1)
            phi_right[i] = reshape(phi_right[i], (-1, n[i] * ry[i + 1]))
            phi_right[i] = phi_right[i][:, ind]

        if dirn > 0 and i < d - 1:
            cry = reshape(cry, (ry[i] * n[i], ry[i + 1]))
            q, r = np.linalg.qr(cry)
            #ind = maxvol(q)
            ind = rect_maxvol(q)[0]
            ry[i + 1] = ind.size
            phi_left[i + 1] = np.tensordot(phi_left[i], cores[i], 1)
            phi_left[i + 1] = reshape(phi_left[i + 1], (ry[i] * n[i], -1))
            phi_left[i + 1] = phi_left[i + 1][ind, :]
            w1 = mkron(np.ones((n[i], 1), dtype=np.int), Jy[i])
            w2 = mkron(grid[i], np.ones((ry[i], 1), dtype=np.int))
            Jy[i + 1] = np.hstack((w1, w2))
            Jy[i + 1] = reshape(Jy[i + 1], (ry[i] * n[i], -1))
            Jy[i + 1] = Jy[i + 1][ind, :]

        i += dirn
        if i == d or i == -1:
            dirn = -dirn
            i += dirn
            swp = swp + 1
    return val, x_full
1296631
def hyper_parameter_search(
            self,
            param_distributions,
            n_iter=10,
            scoring=None,
            fit_params={},
            n_jobs=1,
            iid=True,
            refit=True,
            cv=None,
            verbose=0,
            pre_dispatch='2*njobs',
            random_state=None,
            error_score='raise',
            return_train_score=True,
            test=True,
            score=True,
            save=True,
            custom_data=None
    ):
        
        params = locals()
        params.pop('self')

        self.fitting = lore.metadata.Fitting.create(
            model=self.name,
            custom_data=custom_data,
            snapshot=lore.metadata.Snapshot(pipeline=self.pipeline.name,
                                            head=str(self.pipeline.training_data.head(2)),
                                            tail=str(self.pipeline.training_data.tail(2))
                                            )
        )

        result = RandomizedSearchCV(
            self.estimator,
            param_distributions,
            n_iter=n_iter,
            scoring=scoring,
            n_jobs=n_jobs,
            iid=iid,
            refit=refit,
            cv=cv,
            verbose=verbose,
            pre_dispatch=pre_dispatch,
            random_state=random_state,
            error_score=error_score,
            return_train_score=return_train_score
        ).fit(
            self.pipeline.encoded_training_data.x,
            y=self.pipeline.encoded_training_data.y,
            validation_x=self.pipeline.encoded_validation_data.x,
            validation_y=self.pipeline.encoded_validation_data.y,
            **fit_params
        )
        self.estimator = result.best_estimator_
        self.stats = {}
        self.estimator_kwargs = self.estimator.__getstate__()

        if test:
            self.stats['test'] = self.evaluate(self.pipeline.test_data)

        if score:
            self.stats['score'] = self.score(self.pipeline.test_data)

        self.complete_fitting()

        if save:
            self.save()
        return result
865603
def attention_lm_moe_small():
  
  hparams = attention_lm_moe_base()
  hparams.num_hidden_layers = 4
  hparams.hidden_size = 512
  hparams.filter_size = 2048
  hparams.moe_num_experts = 128
  hparams.moe_layers = "2"
  return hparams
783871
def rlmb_tiny_sv2p():
  """"""
  hparams = rlmb_ppo_tiny()
  hparams.generative_model = "next_frame_sv2p"
  hparams.generative_model_params = "next_frame_sv2p_tiny"
  hparams.grayscale = False
  return hparams
946108
def bytenet_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.batch_size = 2048
  hparams.hidden_size = 768
  hparams.dropout = 0.2
  hparams.symbol_dropout = 0.2
  hparams.label_smoothing = 0.1
  hparams.clip_grad_norm = 2.0
  hparams.num_hidden_layers = 4
  hparams.kernel_height = 3
  hparams.kernel_width = 1
  hparams.learning_rate_decay_scheme = "exp"
  hparams.learning_rate = 0.05
  hparams.learning_rate_warmup_steps = 3000
  hparams.initializer_gain = 1.0
  hparams.weight_decay = 3.0
  hparams.num_sampled_classes = 0
  hparams.sampling_method = "argmax"
  hparams.optimizer_adam_epsilon = 1e-6
  hparams.optimizer_adam_beta1 = 0.85
  hparams.optimizer_adam_beta2 = 0.997
  hparams.add_hparam("num_block_repeat", 4)
  return hparams
1213241
def next_frame_sv2p_tiny():
  """"""
  hparams = next_frame_sv2p_atari_softmax()
  hparams.batch_size = 2
  hparams.tiny_mode = True
  hparams.num_masks = 1
  hparams.video_modality_loss_cutoff = 0.4
  hparams.video_num_input_frames = 4
  hparams.video_num_target_frames = 4
  return hparams
800471
def slicenet_params1_tiny():
  """"""
  hparams = slicenet_params1()
  hparams.attention_type = "simple"
  hparams.separability = 0
  hparams.hidden_size = 128
  hparams.num_hidden_layers = 2
  hparams.batch_size = 512
  hparams.learning_rate_warmup_steps = 200
  return hparams
1174634
private ApproximationLine approximateLowerHull(ConvexHull convexHull, double[] log_k, double sum_log_k, double sum_log_k2, double[] log_kDist, double sum_log_kDist, double sum_log_k_kDist) {
    // StringBuilder msg = new StringBuilder(1000);
    int[] lowerHull = convexHull.getLowerHull();
    int l = convexHull.getNumberOfPointsInLowerHull();
    int k_0 = settings.kmax - lowerHull.length + 1;

    // linear search on all line segments on the lower convex hull
    // msg.append("lower hull l = ").append(l).append('\n');
    double low_error = Double.MAX_VALUE;
    double low_m = 0.0;
    double low_t = 0.0;

    for(int i = 1; i < l; i++) {
      double cur_m = (log_kDist[lowerHull[i]] - log_kDist[lowerHull[i - 1]]) / (log_k[lowerHull[i]] - log_k[lowerHull[i - 1]]);
      double cur_t = log_kDist[lowerHull[i]] - cur_m * log_k[lowerHull[i]];
      double cur_error = ssqerr(k_0, settings.kmax, log_k, log_kDist, cur_m, cur_t);
      // msg.append(" Segment = ").append(i).append(" m =
      // ").append(cur_m).append(" t = ").append(cur_t).append(" lowerror =
      // ").append(cur_error).append('\n');
      if(cur_error < low_error) {
        low_error = cur_error;
        low_m = cur_m;
        low_t = cur_t;
      }
    }

    // linear search on all points of the lower convex hull
    boolean is_right = true; // NEEDED FOR PROOF CHECK
    for(int i = 0; i < l; i++) {
      double cur_m = optimize(k_0, settings.kmax, sum_log_k, sum_log_k2, log_k[lowerHull[i]], log_kDist[lowerHull[i]], sum_log_k_kDist, sum_log_kDist);
      double cur_t = log_kDist[lowerHull[i]] - cur_m * log_k[lowerHull[i]];
      // only valid if both neighboring points are underneath y=mx+t
      if((i == 0 || log_kDist[lowerHull[i - 1]] >= log_kDist[lowerHull[i]] - cur_m * (log_k[lowerHull[i]] - log_k[lowerHull[i - 1]])) && (i == l - 1 || log_kDist[lowerHull[i + 1]] >= log_kDist[lowerHull[i]] + cur_m * (log_k[lowerHull[i + 1]] - log_k[lowerHull[i]]))) {
        double cur_error = ssqerr(k_0, settings.kmax, log_k, log_kDist, cur_m, cur_t);
        if(cur_error < low_error) {
          low_error = cur_error;
          low_m = cur_m;
          low_t = cur_t;
        }
      }

      // check proof of bisection search
      if(!(i > 0 && log_kDist[lowerHull[i - 1]] < log_kDist[lowerHull[i]] - cur_m * (log_k[lowerHull[i]] - log_k[lowerHull[i - 1]])) && !is_right) {
        LOG.warning("ERROR lower: The bisection search will not work properly!");
        if(!(i < l - 1 && log_kDist[lowerHull[i + 1]] < log_kDist[lowerHull[i]] + cur_m * (log_k[lowerHull[i + 1]] - log_k[lowerHull[i]]))) {
          is_right = false;
        }
      }
    }
    return new ApproximationLine(k_0, low_m, low_t);
  }
753285
def _get_hyperparameter_solution_size(meta_parameters):
    
    # WARNING: meta_parameters is modified inline

    solution_size = 0
    for _, parameters in meta_parameters.iteritems():
        if parameters['type'] == 'discrete':
            # Binary encoding of discrete values -> log_2 N
            num_values = len(parameters['values'])
            binary_size = helpers.binary_size(num_values)
        elif parameters['type'] == 'int':
            # Use enough bits to cover range from min to max
            # + 1 to include max in range
            int_range = parameters['max'] - parameters['min'] + 1
            binary_size = helpers.binary_size(int_range)
        elif parameters['type'] == 'float':
            # Use enough bits to provide fine steps between min and max
            float_range = parameters['max'] - parameters['min']
            # * 1000 provides 1000 values between each natural number
            binary_size = helpers.binary_size(float_range * 1000)
        else:
            raise ValueError('Parameter type "{}" does not match known values'.
                             format(parameters['type']))

        # Store binary size with parameters for use in decode function
        parameters['binary_size'] = binary_size

        solution_size += binary_size

    return solution_size
194038
def basic_params1():
  """"""
  return hparam.HParams(
      # If the problem consists of variable-length sequences
      # (see problem.batch_size_means_tokens()), then this is the number
      # of tokens per batch per GPU or per TPU core.  Otherwise, this is
      # the number of examples per GPU or per TPU core.
      batch_size=4096,
      batch_shuffle_size=512,
      # If True, then if the features are of variable length, the batch_size is
      # used as the actual batch size (and not tokens per batch).
      use_fixed_batch_size=False,
      num_hidden_layers=4,
      kernel_height=3,
      kernel_width=1,
      hidden_size=64,
      compress_steps=0,
      # All hyperparameters ending in "dropout" are automatically set to 0.0
      # when not in training mode.
      dropout=0.2,
      clip_grad_norm=2.0,
      grad_noise_scale=0.0,
      summarize_grads=False,
      # Flag for whether mlperf mode is on
      mlperf_mode=False,
      # Whether to log the name and size of every variable
      summarize_vars=False,
      initializer="orthogonal",
      initializer_gain=1.5,
      label_smoothing=0.1,
      optimizer="adam",
      optimizer_adam_epsilon=1e-6,
      optimizer_adam_beta1=0.85,
      optimizer_adam_beta2=0.997,
      optimizer_momentum_momentum=0.9,
      optimizer_momentum_nesterov=False,
      optimizer_adafactor_beta1=0.0,
      optimizer_adafactor_beta2=0.999,
      optimizer_adafactor_factored=True,
      optimizer_adafactor_decay_type="pow",
      optimizer_adafactor_memory_exponent=0.8,
      optimizer_adafactor_clipping_threshold=1.0,
      optimizer_adafactor_multiply_by_parameter_scale=True,
      # Number of accumulating steps for multi step optimizers.
      optimizer_multistep_accumulate_steps=0,
      # Loss scaling used.
      # Generally only necessary with mixed precision training.
      # Mixed precision training only supports exponential scaling currently
      # To disable the scaler, see to 0/False
      mixed_precision_optimizer_loss_scaler="exponential",
      # Determines the initial loss scaling value for mixed precision
      mixed_precision_optimizer_init_loss_scale=2**15,
      # Whether to zero gradients that were not computed, so that the
      # appropriate slots are created. Useful for sharing checkpoints between
      # models with different sets of heads.
      optimizer_zero_grads=False,
      weight_decay=1e-6,
      weight_noise=0.0,
      # Defines the learning rate as a product of named functions.
      # Available functions are listed in learning_rate._LEARNING_RATE_FUNCTIONS
      # e.g. "constant*linear_warmup*rsqrt_decay*rsqrt_hidden_size"
      learning_rate_schedule="legacy",
      learning_rate_constant=1.0,
      # If learning_rate_schedule=="legacy",
      # then we specify decay scheme here.  Warmup is always exponential,
      # except with "noam" learning rate decay scheme.
      # see optimize.legacy_learning_rate_schedule()
      # TODO(noam): migrate everyone away from this.
      learning_rate_decay_scheme="none",
      # decay_steps and decay_staircase for learning_rate_decay_scheme=="exp"
      learning_rate_decay_steps=5000,
      learning_rate_decay_staircase=False,
      learning_rate_minimum=None,
      learning_rate_decay_rate=1.0,
      learning_rate_warmup_steps=100,
      learning_rate_cosine_cycle_steps=250000,
      learning_rate=0.1,
      sampling_method="argmax",  # "argmax" or "random"
      sampling_temp=1.0,  # temperature for sampling
      sampling_keep_top_k=-1,  # If >0, ignore all but the top k logits
      # expand the logits a piece at a time - saves memory.
      factored_logits=False,
      multiply_embedding_mode="sqrt_depth",
      # Parameters related to mixtures of experts.
      moe_hidden_sizes="2048",  # hidden layer sizes (comma-separated)
      moe_num_experts=64,  # number of experts per layer
      moe_k=2,  # how many experts to use for each batch element
      moe_loss_coef=1e-2,
      # Sequences of operations to perform on layer input and layer output.
      # Used by common_layers.layer_preprocess, common_layers.layer_postprocess
      # Each character represents an operation:
      # none: no preprocessing
      #    d: apply dropout
      #    n: apply normalization (see norm_type and norm_epsilon)
      #    a: add layer input (residual connection - only during postprocess)
      # The special string "none" is used instead of the empty string
      # to indicate no pre/postprocessing, since the empty string causes
      # trouble for hyperparameter tuning.
      # TODO(noam): The current settings ("", "dan") are the published version
      # of the transformer.  ("n", "da") seems better for harder-to-learn
      # models, so it should probably be the default.
      layer_preprocess_sequence="none",
      layer_postprocess_sequence="dan",
      # dropout rate to use during layer_preprocess and layer_postprocess
      layer_prepostprocess_dropout=0.1,
      # broadcast dimensions for layer_prepostprocess_dropout
      # a comma-separated list of integers.
      # see common_layers.dropout_with_broadcast_dims()
      # Change this to "1" to save memory.
      layer_prepostprocess_dropout_broadcast_dims="",
      # dropout some symbols (set them to 0) before embedding.
      symbol_dropout=0.0,
      # What type of normalization to use
      norm_type="layer",  # "batch", layer", "noam", "none".
      # epsilon parameter to normalization function
      norm_epsilon=1e-6,
      # pad vocabularies so that this value divides the vocabulary size.
      vocab_divisor=1,
      # During training, we drop sequences whose inputs and targets are shorter
      # than min_length
      min_length=0,
      # During training, we drop sequences whose inputs or targets are longer
      # than max_length.
      # If max_length==0, we use hparams.batch_size instead.
      max_length=0,
      # Pack examples on the fly.
      pack_dataset=False,
      # Use custom ops not included in standard tensorflow.
      use_custom_ops=True,
      # Split targets on the first axis into chunks of this length.
      split_targets_chunk_length=0,
      split_targets_max_chunks=100,
      split_targets_strided_training=False,
      # Maximum length in the smallest length bucket.  Setting this
      # flag too high will result in wasteful padding of short
      # sequences.  Due to some (hopefully) temporary hacks in the
      # data reading and batching code, setting this flag too low
      # results in a very long batch-shuffling queue.
      # TODO(noam): change this once the Datasets API changes.
      min_length_bucket=8,
      # This flag controls the number of length buckets in the data
      # reader.  The buckets have maximum lengths from
      # min_bucket_length to (max_length or batch_size), increasing
      # (approximately) by factors of length_bucket_step.
      length_bucket_step=1.1,
      # If set to True, drop sequences longer than max_length during eval.
      # This affects the validity of the evaluation metrics.
      eval_drop_long_sequences=False,
      # If True, run the model autoregressively instead of teacher-forcing
      # during eval
      eval_run_autoregressive=False,
      # (For features with symbol modality) If True, share all of the
      # input embeddings, target embeddings, and softmax weights.
      shared_embedding_and_softmax_weights=False,
      # (For features with symbol modality) If True, share the input embeddings
      # and target embeddings.
      shared_embedding=False,
      # (For features with symbol modality) Number to shard embeddings by.
      symbol_modality_num_shards=1,
      # Feature transformations are optional dictionaries comprising key-value
      # pairs of a feature name (str) and its transformation (function). If not
      # specified, T2TModel applies a default transformation according to the
      # feature's modality. Bottom is applicable to all features; loss, top, and
      # weights_fn are only applicable to target features.
      # TODO(trandustin): `name` is an optional hparam for legacy reasons,
      # defining variable scope names. Remove this hparam in the future.
      bottom={},
      loss={},
      name={},
      top={},
      weights_fn={},
      # The maximum length of "input" sequence.
      # Sequences longer than this value will be truncated. 0 or negative values
      # mean there is no maximum or truncation.
      # You can change this behavior by overriding preprocess_example() method
      # in your problem class.
      max_input_seq_length=0,
      # The maximum length of "target" sequence.
      # Sequences longer than this value will be truncated. 0 or negative values
      # mean there is no maximum or truncation.
      # You can change this behavior by overriding preprocess_example() method
      # in your problem class.
      max_target_seq_length=0,
      # if nonzero, we split the target sequences on example read.
      # This is for use with language modeling problems with fixed length
      # examples.  e.g.  The examples may be written with length 65536, but we
      # want to split each example into 64 examples of length 1024.
      split_to_length=0,
      # Video settings: how many frames to batch on input and targets.
      video_num_input_frames=1,
      video_num_target_frames=1,
      # This flag allows us to optionally treat a seq-to-seq problem
      # as a language model.  Legal values are:
      #
      # "none" - Do not prepend the inputs to the targets.
      # "prepend_inputs_masked_attention"
      #     replace "targets" in preprocessing with
      #     tf.concat([inputs, [0], targets], axis=1)
      #     i.e. we prepend the inputs to the targets with a single
      #     padding token in between.  Use masked self-attention on the
      #     entire resulting sequence.  During training, we compute losses on
      #     the combined sequence.  During eval, we compute the metrics
      #     on only the targets portion.
      # "prepend_inputs_full_attention"
      #     similar to the previous option except that each
      #     position in the inputs portion can see the
      #     entire inputs portion.  This removes the challenge of
      #     autoregressively predicting the inputs portion.
      prepend_mode="none",
      # Scheduled sampling is interesting for auto-regressive models.
      # It runs an additional step using the generated output as autoregressive
      # targets, which can improve the models inference results later. The
      # parameter scheduled_sampling_prob determines with what probability
      # will such additional step be run. It's turned off (0.0) by default.
      # This probability will exponentially warm up for the number of
      # steps determined by scheduled_sampling_warmup_steps.
      # The tensor used for the n-th pass will consist of outputs from
      # the (n-1)-th pass mixed with gold truth, with the proportion of gold
      # determined by scheduled_sampling_gold_mixin_prob. Control the number
      # of passes with scheduled_sampling_num_passes.
      scheduled_sampling_prob=0.0,
      scheduled_sampling_warmup_steps=50000,
      scheduled_sampling_gold_mixin_prob=0.5,
      # TODO(duckworthd): Uncomment when we can ascertain why adding an
      # extra field to HParam causes test failures.
      # scheduled_sampling_num_passes=1,

      # This setting controls whether to copy variables around in a daisy chain
      # (if true) or leave their placement to TensorFlow. It only affects multi
      # device training and mostly should be turned on for performance. One
      # exception are recurrent models: with dynamic loops it must be off.
      daisy_chain_variables=True,
      # If True in PREDICT mode, then last-position-only optimizations are not
      # used.
      force_full_predict=False,
      # Set this for pure model parallelism.  There is only one data shard.
      no_data_parallelism=False,
      # dtype used for activations. - "float32" or "bfloat16"
      # activation_dtype="bfloat16" currently only works on TPU.
      #    It lowers activation-memory usage
      #    and does not appear to affect quality.
      #    You can train on TPU with activation_dtype="bfloat16" and evaluate
      #    on CPU/GPU with activation_dtype="float32"
      activation_dtype="float32",
      # dtype used for parameters: "float32" or "bfloat16"
      # bfloat16 currently only works with optimizer="adafactor".
      #   The savings in memory allow for training larger models.
      #   Weights are encoded as (w*128)^8, using pseudostochastic
      #   roundoff.  Initial experiments show that model quality is similar
      #   to baseline for about 3M training steps, but worse thereafter.
      weight_dtype="float32",
      # Directory containing a checkpoint for a pretrained model. This will only
      # be used if a new run is being started. Parameters not found in the
      # pretrained model will be randomly initialized. Superfluous parameters in
      # the pretrained model will be ignored.
      pretrained_model_dir="",
      # Threshold used for two cases: the primary task probability for the
      # constant mixing schedule, and the exponential schedule limit for when
      # mixing should stop (eg: 0.5 means stop at 50-50 mixing, 0.8 means stop
      # at 20-80 mixing for the primary-others mixing case.)
      multiproblem_schedule_threshold=0.5,
      # For more than 2 tasks, we may want to specify per-task thresholds here.
      # In that case, this needs to be a string with as many floating point
      # numbers as the number of tasks in the multi-problem. These numbers
      # are later normalized to add up to 1 and taken as probabilities for
      # each task. This enforces a constant mixing schedule and if this is
      # empty then the threshold from above is used for the first task and
      # the other tasks get the remaining probability split uniformly.
      multiproblem_per_task_threshold="",
      # The number of examples at which the proportion of the mixed in datasets
      # is multiproblem_schedule_threshold
      multiproblem_schedule_max_examples=1e7,
      # When training multiproblems, we can mix the data according to different
      # schedules. Example: a constant schedule mixing 20-80 between the primary
      # and other tasks.
      # A list of supported schedules can be found in
      # `data_generators.multi_problem.py`.
      multiproblem_mixing_schedule="constant",
      # A boolean that decides whether input sequence losses and target label
      # losses in classification problems should be reweighted.
      multiproblem_reweight_label_loss=False,
      # How much weight the targets in classification problems receive. Inputs
      # receive 1 minus this weight.
      multiproblem_label_weight=0.5,
      # Hyperparameters for relative attention.
      # The maximum relative positional distance to learn an embedding for.
      max_relative_position=0,
      # If heads share the same relative embedding.
      heads_share_relative_embedding=False,
      # If relative embedding terms are added to values too.
      add_relative_to_values=False,
      # If enable the host_call which is executed every training step.
      # There could be a performance drop if host_call function is slow and
      # cannot keep up with the TPU-side computation.
      tpu_enable_host_call=False,
      # Pad batch dim of inputs to nearest multiple of batch multiple.
      pad_batch=False,
      # When true, do not evaluate on the language model data when running the
      # multiproblem since it can take a while. If False, set eval_steps to
      # something large like 6000 or 10000.
      multiproblem_target_eval_only=False,
      # Max out the vocab size to a power of 2 for efficiency and to reserve
      # extra space in the vocabulary for new task ids and label classes.
      multiproblem_vocab_size=-1,
      # When using multiproblem with generation tasks, need to truncate the
      # inputs and targets manually before concatenating them.
      multiproblem_max_input_length=-1,
      multiproblem_max_target_length=-1,
      # If positive, makes training targets fixed-length in MultiProblem.
      multiproblem_fixed_train_length=-1,
      # Load weights from a second model. For instance, when using
      # pre-trained weights, you might want to initialize the encoder
      # and decoder by loading different models.
      warm_start_from_second="",
      # Area attention hyper parameters
      area_value_mode="none",
      area_key_mode="none",
      # Using area attention for the number of layers from the bottom
      num_area_layers=0,
      max_area_width=1,
      max_area_height=1,
      memory_height=1
  )
417062
def update_hparams_for_tpu(hparams):
  """"""

  # Adafactor uses less memory than Adam.
  # switch to Adafactor with its recommended learning rate scheme.
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_schedule = "rsqrt_decay"
  hparams.learning_rate_warmup_steps = 10000

  # Avoid an expensive concat on TPU.
  # >1 shards helps with faster parameter distribution on multi-GPU machines
  hparams.symbol_modality_num_shards = 1

  # Adaptive batch sizes and sequence lengths are not supported on TPU.
  # Instead, every batch has the same sequence length and the same batch size.
  # Longer sequences are dropped and shorter ones are padded.
  #
  # It is therefore suggested to use a problem where examples have been combined
  # to a longer length, e.g. the "_packed" problems.
  #
  # For problems with variable sequence lengths, this parameter controls the
  # maximum sequence length.  Shorter sequences are dropped and longer ones
  # are padded.
  #
  # For problems with fixed sequence lengths - e.g. the "_packed" problems,
  # this hyperparameter is ignored.
  hparams.max_length = 64

  # TPUs have less memory than GPUs, so decrease the batch size
  hparams.batch_size = 2048

  # Using noise broadcast in the dropout layers saves memory during training.
  hparams.attention_dropout_broadcast_dims = "0,1"  # batch, heads
  hparams.relu_dropout_broadcast_dims = "1"  # length
  hparams.layer_prepostprocess_dropout_broadcast_dims = "1"  # length
  return hparams
1075626
def transformer_base_vq_ada_32ex_packed():
  """"""
  hparams = transformer_base_v2()
  expert_utils.update_hparams_for_vq_gating(hparams)
  hparams.moe_num_experts = 32
  hparams.gating_type = "vq"
  # this gives us a batch size of 16 because each seq is len 256
  hparams.batch_size = 5072
  hparams.ffn_layer = "local_moe"
  hparams.shared_embedding_and_softmax_weights = False
  hparams.learning_rate_warmup_steps = 10000
  # one epoch for languagemodel_lm1b32k_packed = 27200 steps w/ bsize 128
  hparams.learning_rate_decay_steps = 27200
  hparams.num_heads = 4
  hparams.num_blocks = 1
  hparams.moe_k = 1
  hparams.num_decoder_layers = 6
  hparams.label_smoothing = 0.
  hparams.layer_prepostprocess_dropout = 0.1
  hparams.layer_postprocess_sequence = "dan"
  hparams.layer_preprocess_sequence = "none"
  hparams.weight_decay = 1e-06
  hparams.attention_dropout = 0.1
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_schedule = "linear_warmup*rsqrt_decay*linear_decay"
  hparams.activation_dtype = "float32"
  hparams.learning_rate = 0.1
  hparams.learning_rate_constant = 1.0
  return hparams
1253887
def mtf_image_transformer_base_imagenet_mp():
  """"""
  hparams = mtf_image_transformer_base_imagenet()
  hparams.mesh_shape = "model:4;batch:8"
  hparams.layout = "batch:batch;d_ff:model;heads:model"
  hparams.batch_size = 32
  hparams.num_heads = 8
  hparams.d_ff = 8192
  hparams.learning_rate_warmup_steps = 31250
  hparams.unconditional = True
  return hparams
1020602
def vq_discrete_bottleneck(x, hparams):
  """"""
  tf.logging.info("Using EMA with beta = {}".format(hparams.beta))
  bottleneck_size = 2**hparams.bottleneck_bits
  x_shape = common_layers.shape_list(x)
  x = tf.reshape(x, [-1, hparams.hidden_size])
  x_means_hot, e_loss = vq_nearest_neighbor(
      x, hparams)
  means, ema_means, ema_count = (hparams.means, hparams.ema_means,
                                 hparams.ema_count)

  # Update the ema variables
  updated_ema_count = moving_averages.assign_moving_average(
      ema_count,
      tf.reduce_sum(x_means_hot, axis=0),
      hparams.decay,
      zero_debias=False)

  dw = tf.matmul(x_means_hot, x, transpose_a=True)
  updated_ema_means = moving_averages.assign_moving_average(
      ema_means, dw, hparams.decay, zero_debias=False)
  n = tf.reduce_sum(updated_ema_count, axis=-1, keepdims=True)
  updated_ema_count = (
      (updated_ema_count + hparams.epsilon) /
      (n + bottleneck_size * hparams.epsilon) * n)
  # pylint: disable=g-no-augmented-assignment
  updated_ema_means = updated_ema_means / tf.expand_dims(
      updated_ema_count, axis=-1)
  # pylint: enable=g-no-augmented-assignment
  with tf.control_dependencies([e_loss]):
    update_means = tf.assign(means, updated_ema_means)
    with tf.control_dependencies([update_means]):
      loss = hparams.beta * e_loss

  discrete = tf.reshape(x_means_hot, x_shape[:-1] + [bottleneck_size])
  return discrete, loss