Hparams for transformer on LM for finetuning on text class problems.
185959
def transformer_tall_finetune_textclass():
  """"""
  hparams = transformer_tall()
  hparams.learning_rate_constant = 6.25e-5
  hparams.learning_rate_schedule = ("linear_warmup*constant*linear_decay")
  hparams.multiproblem_schedule_max_examples = 0
  hparams.multiproblem_target_eval_only = True
  hparams.learning_rate_warmup_steps = 50
  # Set train steps to learning_rate_decay_steps or less
  hparams.learning_rate_decay_steps = 25000
  hparams.multiproblem_reweight_label_loss = True
  hparams.multiproblem_label_weight = 0.95
  return hparams
459208
def add_librispeech_hparams(hparams):
  """"""
  hparams.batch_size = 36
  hparams.audio_compression = 8
  hparams.hidden_size = 2048
  hparams.max_input_seq_length = 600000
  hparams.max_target_seq_length = 350
  hparams.max_length = hparams.max_input_seq_length
  hparams.min_length_bucket = hparams.max_input_seq_length // 2
  hparams.learning_rate = 0.05
  hparams.train_steps = 5000000
  hparams.num_hidden_layers = 4
  return hparams
1262411
def transformer_encoder_layers(inputs,
                               num_layers,
                               hparams,
                               attention_type=AttentionType.GLOBAL,
                               self_attention_bias=None,
                               q_padding="VALID",
                               kv_padding="VALID",
                               name="transformer"):
  """"""
  x = inputs
  x = tf.nn.dropout(x, 1.0 - hparams.layer_prepostprocess_dropout)

  for layer in range(num_layers):
    # attention layers + skip connections
    with tf.variable_scope("%s_layer_%d" % (name, layer)):
      if attention_type == AttentionType.LOCAL_2D:
        y = local_attention_2d(common_layers.layer_preprocess(x, hparams),
                               hparams,
                               attention_type="local_attention_2d")
      elif attention_type == AttentionType.LOCAL_1D:
        y = local_attention_1d(common_layers.layer_preprocess(x, hparams),
                               hparams,
                               attention_type="local_unmasked",
                               q_padding=q_padding, kv_padding=kv_padding)
      elif attention_type == AttentionType.GLOBAL:
        y = full_self_attention(common_layers.layer_preprocess(x, hparams),
                                self_attention_bias, hparams,
                                q_padding=q_padding, kv_padding=kv_padding)
      x = common_layers.layer_postprocess(x, y, hparams)
      # feed-fwd layer + skip connections
      y = ffn_layer(common_layers.layer_preprocess(x, hparams), hparams)
      x = common_layers.layer_postprocess(x, y, hparams)
  return common_layers.layer_preprocess(x, hparams)
1247157
def adaptive_universal_transformer_multilayer_hard():
  """"""
  hparams = adaptive_universal_transformer_multilayer_tpu()
  hparams.batch_size = 256
  hparams.hard_attention_k = 8
  hparams.add_step_timing_signal = True
  # hparams.add_sru = True  # This is very slow on GPUs, does it help?
  hparams.self_attention_type = "dot_product_relative_v2"
  hparams.max_relative_position = 256
  return hparams
691021
def transformer_librispeech_tpu_v1():
  """"""
  hparams = transformer_librispeech_v1()
  update_hparams_for_tpu(hparams)

  hparams.batch_size = 16
  librispeech.set_librispeech_length_hparams(hparams)
  return hparams
728476
def normalize_example(self, example, hparams):
    """"""

    length = self.max_length(hparams)
    def _to_constant_shape(tensor):
      tensor = tensor[:length]
      tensor = tf.pad(tensor, [(0, length - tf.shape(tensor)[0])])
      return tf.reshape(tensor, [length])

    if self.has_inputs:
      example['inputs'] = _to_constant_shape(example['inputs'])
      example['targets'] = _to_constant_shape(example['targets'])
    elif 'inputs' in example:
      if self.packed_length:
        raise ValueError('cannot concatenate packed examples on the fly.')
      inputs = example.pop('inputs')[:-1]  # Remove EOS token.
      targets = tf.concat([inputs, example['targets']], 0)
      example['targets'] = _to_constant_shape(targets)
    else:
      example['targets'] = _to_constant_shape(example['targets'])
    if self.packed_length:
      if self.has_inputs:
        if 'inputs_segmentation' in example:
          example['inputs_segmentation'] = _to_constant_shape(
              example['inputs_segmentation'])
          example['inputs_position'] = _to_constant_shape(
              example['inputs_position'])
        else:
          example['inputs_segmentation'] = tf.to_int64(
              tf.not_equal(example['inputs'], 0))
          example['inputs_position'] = (
              example['inputs_segmentation'] * tf.range(length, dtype=tf.int64))
      if 'targets_segmentation' in example:
        example['targets_segmentation'] = _to_constant_shape(
            example['targets_segmentation'])
        example['targets_position'] = _to_constant_shape(
            example['targets_position'])
      else:
        example['targets_segmentation'] = tf.to_int64(
            tf.not_equal(example['targets'], 0))
        example['targets_position'] = (
            example['targets_segmentation'] * tf.range(length, dtype=tf.int64))
    return example
1300807
def attention_lm_moe_base():
  
  hparams = common_hparams.basic_params1()
  hparams.hidden_size = 1024
  hparams.batch_size = 8192
  hparams.max_length = 256
  hparams.dropout = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 2000
  hparams.initializer_gain = 1.0
  hparams.num_hidden_layers = 4
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.98
  hparams.num_sampled_classes = 0
  hparams.label_smoothing = 0.0
  hparams.shared_embedding_and_softmax_weights = False
  hparams.add_hparam("filter_size", 2048)  # Add new ones like this.
  hparams.moe_num_experts = 32
  # attention-related flags
  hparams.add_hparam("num_heads", 8)
  hparams.add_hparam("attention_key_channels", 0)
  hparams.add_hparam("attention_value_channels", 0)
  # All hyperparameters ending in "dropout" are automatically set to 0.0
  # when not in training mode.
  hparams.add_hparam("attention_dropout", 0.0)
  hparams.add_hparam("relu_dropout", 0.0)
  hparams.add_hparam("pos", "timing")  # timing, none
  hparams.add_hparam("moe_layers", "2")  # comma separated list of layer numbers
  # moe params. local attention moe.
  # If attention_layers is set, the num_hidden_layers parameter will be ignored
  # and each caracter of the string will correspond to one attention
  # layer type
  hparams.add_hparam("attention_layers", "")
  hparams.add_hparam("attention_type", AttentionType.MULTIHEAD)
  hparams.add_hparam("attention_local", False)
  hparams.add_hparam("attention_moe_k", 2)
  hparams.add_hparam("attention_num_head", 1)
  hparams.add_hparam("attention_num_experts", 16)
  hparams.add_hparam("attention_split_batch", False)
  hparams.add_hparam("attention_red_factor", 3)
  hparams.add_hparam("attention_block_length", 128)
  hparams.add_hparam("attention_reduction_type", "conv")
  # Non linearity for the attention reduction. Either "none", or "silu" (
  # Sigmoid Linear-Unit described in https://arxiv.org/abs/1710.05941)
  hparams.add_hparam("attention_nonlinearity", "none")
  # If attention_exp_factor is set, each input to local_expert_attention (of
  # dimensionality hidden size) is projected into attention_exp_factor smaller
  # inputs, each of dimensionality attention_exp_inputdim. (otherwise
  # attention_exp_inputdim is ignored)
  hparams.add_hparam("attention_exp_factor", 0)
  hparams.add_hparam("attention_exp_inputdim", 128)
  # Key, query and value dimensions for the attention
  hparams.add_hparam("attention_kq_size", 128)
  hparams.add_hparam("attention_v_size", 256)
  # Loss coef for load balancing
  hparams.add_hparam("attention_load_balance", 2e-2)
  # Locality-sensitive hashing params
  hparams.add_hparam("lsh_num_hyperplanes", 4)
  hparams.add_hparam("lsh_use_map_fn", False)

  hparams.add_hparam("use_sepconv", False)
  hparams.add_hparam("diet_experts", False)
  hparams.add_hparam("memory_efficient_ffn", False)
  # if True, we learn a non-autoregressive model from "inputs" to "targets".
  # if False, we learn an autoregressive model to generate "targets"
  hparams.add_hparam("use_inputs", False)
  return hparams
49763
def evolved_transformer_big_tpu():
  """"""
  hparams = add_evolved_transformer_hparams(transformer.transformer_big_tpu())
  hparams.learning_rate_constant = 1 / hparams.learning_rate_warmup_steps ** 0.5
  hparams.learning_rate_schedule = (
      "constant*single_cycle_cos_decay")
  return hparams
1064209
def img2img_transformer2d_n31():
  """"""
  hparams = img2img_transformer2d_base()
  hparams.batch_size = 1
  hparams.num_encoder_layers = 6
  hparams.num_decoder_layers = 12
  hparams.num_heads = 8
  hparams.query_shape = (16, 32)
  hparams.memory_flange = (16, 32)
  return hparams
885254
def parse_args():
    """"""
    parser = argparse.ArgumentParser(
        description='Text Classification with FastText',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # Computation options
    group = parser.add_argument_group('Computation arguments')
    group.add_argument('--input', type=str, help='Input file location')
    group.add_argument(
        '--validation', type=str, help='Validation file Location ')
    group.add_argument(
        '--output', type=str, help='Location to save trained model')
    group.add_argument(
        '--ngrams', type=int, default=1, help='NGrams used for training')
    group.add_argument(
        '--batch_size', type=int, default=16, help='Batch size for training.')
    group.add_argument('--epochs', type=int, default=10, help='Epoch limit')
    group.add_argument(
        '--gpu',
        type=int,
        help=('Number (index) of GPU to run on, e.g. 0. '
              'If not specified, uses CPU.'))
    group.add_argument(
        '--no-hybridize',
        action='store_true',
        help='Disable hybridization of gluon HybridBlocks.')

    # Model
    group = parser.add_argument_group('Model arguments')
    group.add_argument(
        '--emsize', type=int, default=100, help='Size of embedding vectors.')

    # Optimization options
    group = parser.add_argument_group('Optimization arguments')
    group.add_argument('--optimizer', type=str, default='adam')
    group.add_argument('--lr', type=float, default=0.05)

    args = parser.parse_args()
    return args
750425
def transformer_nat_small():
  """"""
  hparams = transformer.transformer_small()
  hparams.batch_size = 2048
  hparams.learning_rate = 0.2
  hparams.learning_rate_warmup_steps = 4000
  hparams.num_hidden_layers = 3
  hparams.hidden_size = 384
  hparams.filter_size = 2048
  hparams.label_smoothing = 0.0
  hparams.force_full_predict = True
  hparams.optimizer = "adam"
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.997
  hparams.add_hparam("bottleneck_kind", "vq")
  hparams.add_hparam("bottleneck_bits", 12)
  hparams.add_hparam("num_compress_steps", 3)
  hparams.add_hparam("beta", 0.25)
  hparams.add_hparam("epsilon", 1e-5)
  hparams.add_hparam("decay", 0.999)
  hparams.add_hparam("num_samples", 10)
  hparams.add_hparam("mask_startup_steps", 50000)
  return hparams
1268426
def transformer_big():
  """"""
  hparams = transformer_base()
  hparams.hidden_size = 1024
  hparams.filter_size = 4096
  # Reduce batch size to 2048 from 4096 to be able to train the model on a GPU
  # with 12 GB memory. For example, NVIDIA TITAN V GPU.
  hparams.batch_size = 2048
  hparams.num_heads = 16
  hparams.layer_prepostprocess_dropout = 0.3
  return hparams
900264
def ae_transformer_internal(inputs,
                            targets,
                            target_space,
                            hparams,
                            cache=None,
                            predict_mask=1.0):
  """"""
  # Summaries break with the do_refine cond, turn them off in that case.
  global _DO_SUMMARIES
  if hparams.do_refine:
    _DO_SUMMARIES = False

  # Prepare.
  if inputs is not None:
    batch_size = common_layers.shape_list(inputs)[0]
  else:
    batch_size = common_layers.shape_list(targets)[0]
  targets = tf.reshape(targets, [batch_size, -1, 1, hparams.hidden_size])

  # Encoder.
  if inputs is not None:
    inputs = common_layers.flatten4d3d(inputs)
    inputs, ed = encode(inputs, target_space, hparams, "input_enc")
    inputs_ex, ed_ex = inputs, ed
  else:
    ed, inputs_ex, ed_ex = None, None, None

  # Autoencoding.
  losses = {"extra": tf.constant(0.0), "latent_pred": tf.constant(0.0),
            "neg_q_entropy": tf.constant(0.0)}
  if hparams.do_ae:
    # flatten here
    original_targets = targets
    original_targets_shape = tf.shape(original_targets)
    if hparams.task == "image":
      cia.maybe_reshape_4d_to_3d(targets)
    if hparams.task == "translate":
      if inputs is not None:
        max_targets_len_from_inputs = tf.concat([inputs, inputs], axis=1)
      else:
        max_targets_len_from_inputs = targets
    else:
      assert hparams.task == "image"
      max_targets_len_from_inputs = targets
    if hparams.word_shuffle:
      tf.logging.info("Using word shuffle with rate = {}".format(
          hparams.word_shuffle))
      targets_idx = tf.range(start=0,
                             limit=common_layers.shape_list(targets)[1],
                             delta=1)
      targets_idx = tf.to_float(targets_idx)
      noise = tf.random_uniform(shape=common_layers.shape_list(targets_idx),
                                minval=0,
                                maxval=1 + hparams.word_shuffle)
      targets_idx += noise
      permutation = tf.contrib.framework.argsort(targets_idx)
      targets_permuted = tf.gather(targets, indices=permutation, axis=1)
      targets = targets_permuted
    targets, _ = common_layers.pad_to_same_length(
        targets, max_targets_len_from_inputs,
        final_length_divisible_by=2**hparams.num_compress_steps)
    # Add positional information
    targets_shape = common_layers.shape_list(targets)
    targets = tf.reshape(targets, [targets_shape[0], targets_shape[1],
                                   targets_shape[3]])
    targets = common_attention.add_positional_embedding(
        targets, hparams.max_length, name="targets_position")
    targets = tf.reshape(targets, shape=targets_shape)
    if hparams.word_dropout:
      mask = tf.random_uniform(shape=common_layers.shape_list(targets),
                               minval=0.0, maxval=1.0)
      targets_noisy = tf.where(mask > hparams.word_dropout, targets,
                               tf.zeros_like(targets))
    else:
      targets_noisy = targets

    targets_c = compress(targets_noisy, inputs, False, hparams, "compress")
    if hparams.mode != tf.estimator.ModeKeys.PREDICT:
      # Compress and bottleneck.
      latents_dense, latents_discrete, extra_loss, embed, neg_q_entropy = (
          hparams.bottleneck(inputs=targets_c,
                             filter_size=hparams.compress_filter_size,
                             mode=hparams.mode,
                             name="vc"))
      if _DO_SUMMARIES:
        tf.summary.histogram("b0", tf.reshape(latents_discrete[:, 0, :], [-1]))
      pc = common_layers.inverse_exp_decay(hparams.startup_steps)
      pc = pc if hparams.mode == tf.estimator.ModeKeys.TRAIN else 1.0
      cond = tf.less(tf.random_uniform([batch_size]), pc)
      latents_dense = tf.where(cond, latents_dense, targets_c)
      # TODO(lukaszkaiser): return extra losses batchwise, multiply before mean.
      losses["extra"] = extra_loss * tf.reduce_mean(tf.to_float(cond))
      # Extra loss predicting latent code from input. Discrete only.
      if hparams.bottleneck_kind not in ["dense", "vae"]:
        latents_pred = decode_transformer(
            inputs_ex, ed_ex,
            embed(latents_discrete), hparams, "extra",
            task="translate")
        _, latent_pred_loss = ae_latent_softmax(
            latents_pred, tf.stop_gradient(latents_discrete), hparams)

        # Scale by latent dimension for summary so we can compare across
        # batches.
        if _DO_SUMMARIES:
          tf.summary.scalar("latent_pred_loss_mean",
                            tf.reduce_mean(latent_pred_loss))
        if hparams.sum_over_latents:
          latent_pred_loss = tf.reduce_sum(latent_pred_loss, [1, 2])

        losses["latent_pred"] = tf.reduce_mean(
            latent_pred_loss * tf.to_float(cond)) * hparams.prior_scale
        losses["neg_q_entropy"] = neg_q_entropy * hparams.entropy_scale
      else:
        inputs_c = decode_transformer(inputs, ed, targets_c, hparams, "dec_c")
        losses["latent_pred"] = tf.reduce_mean(
            tf.squared_difference(inputs_c, targets_c)) * 20
        def bn_inputs():
          with tf.variable_scope(tf.get_variable_scope(), reuse=True):
            bn, _, _, _, _ = hparams.bottleneck(
                inputs=inputs_c,
                filter_size=hparams.compress_filter_size,
                mode=hparams.mode,
                name="vc")
          return bn
        inputs_c = bn_inputs()
        ptc = 1.0 - common_layers.inverse_lin_decay(200000) * 0.5
        ptc = ptc if hparams.mode == tf.estimator.ModeKeys.TRAIN else 1.0
        latents_dense = tf.where(tf.less(tf.random_uniform([batch_size]), ptc),
                                 latents_dense, inputs_c)
    else:
      if hparams.bottleneck_kind in ["dense", "vae"]:
        inputs_c = decode_transformer(inputs, ed, targets_c, hparams, "dec_c")
        latents_dense, _, _, _, _ = hparams.bottleneck(
            inputs=inputs_c,
            filter_size=hparams.compress_filter_size,
            mode=hparams.mode,
            name="vc")
      else:
        latent_len = common_layers.shape_list(targets_c)[1]
        _, _, _, embed, _ = hparams.bottleneck(
            inputs=targets_c,
            filter_size=hparams.compress_filter_size,
            name="vc")
        latents_dense = tf.zeros_like(targets_c[:, :latent_len, :, :])
        if cache is None:
          cache = ae_latent_sample(
              latents_dense, inputs_ex, ed_ex, embed, 16, hparams)
        latents_dense = embed(cache)
    # Postprocess.
    d = latents_dense
    d_shape = common_layers.shape_list(d)
    d = tf.reshape(d, [d_shape[0], d_shape[1], d_shape[3]])
    d = common_attention.add_positional_embedding(
        d, hparams.max_length, name="latents_position")
    d = tf.reshape(d, shape=d_shape)

    # decompressing the dense latents
    for i in range(hparams.num_compress_steps):
      j = hparams.num_compress_steps - i - 1
      d = residual_conv(d, 1, (3, 1), hparams, "decompress_rc_%d" % j)
      if inputs is not None and hparams.do_attend_decompress:
        d = attend(d, inputs, hparams, "decompress_attend_%d" % j)
      d = decompress_step(d, hparams, i > 0, False, "decompress_%d" % j)

    # Masking.
    if hparams.do_mask:
      masking = common_layers.inverse_lin_decay(hparams.mask_startup_steps)
      masking *= common_layers.inverse_exp_decay(
          hparams.mask_startup_steps // 4)  # Not much at start.
      if not hparams.do_refine:
        masking -= tf.random_uniform([]) * hparams.unmasked_percentage
      masking = tf.minimum(tf.maximum(masking, 0.0), 1.0)
      if hparams.use_predict_mask:
        masking = predict_mask
      if hparams.mode == tf.estimator.ModeKeys.PREDICT:
        masking = predict_mask
      mask = tf.less(masking, tf.random_uniform(
          common_layers.shape_list(targets)[:-1]))
      mask = tf.expand_dims(tf.to_float(mask), 3)

      # targets is always [batch, length, 1, depth]
      targets = mask * targets + (1.0 - mask) * d
      # reshape back to 4d here
      if hparams.task == "image":
        targets = tf.reshape(targets, original_targets_shape)

  res = decode_transformer(inputs, ed, targets, hparams, "decoder",
                           causal=hparams.causal)
  if hparams.do_ae:
    if hparams.do_mask and hparams.do_refine:
      def refine_res():
        # return residual_conv(res, 1, (5, 1), hparams, "refine")
        r, _ = encode(tf.squeeze(res, axis=[2]),
                      target_space, hparams, "refine_enc")
        return tf.expand_dims(r, axis=2)
      masked_batches = tf.reduce_sum(mask, axis=[1, 2, 3])
      all_masked = tf.less(masked_batches, 0.1)
      res = tf.where(all_masked, refine_res(), res)
    # We'll start training the extra model of latents after mask_startup_steps.
    nonlatent_steps = hparams.mask_startup_steps
    latent_time = tf.less(nonlatent_steps,
                          tf.to_int32(tf.train.get_global_step()))
    losses["latent_pred"] *= tf.to_float(latent_time)

  # res was generated from padded targets, which means it has some extra
  # elements. These can cause shape problems when computing loss with respect to
  # the original (unpadded) targets. So we remove their extra elements here.
  res = res[:, :original_targets_shape[1], :, :]

  data_dim = common_layers.shape_list(res)[1]
  latent_dim = common_layers.shape_list(targets_c)[1]
  return res, losses, cache, data_dim, latent_dim
639701
def preprocess_data(tokenizer, task, batch_size, dev_batch_size, max_len, pad=False):
    """"""
    # transformation
    trans = BERTDatasetTransform(
        tokenizer,
        max_len,
        labels=task.get_labels(),
        pad=pad,
        pair=task.is_pair,
        label_dtype='float32' if not task.get_labels() else 'int32')

    data_train = task('train').transform(trans, lazy=False)
    data_train_len = data_train.transform(
        lambda input_id, length, segment_id, label_id: length)

    num_samples_train = len(data_train)
    # bucket sampler
    batchify_fn = nlp.data.batchify.Tuple(
        nlp.data.batchify.Pad(axis=0), nlp.data.batchify.Stack(),
        nlp.data.batchify.Pad(axis=0),
        nlp.data.batchify.Stack(
            'float32' if not task.get_labels() else 'int32'))
    batch_sampler = nlp.data.sampler.FixedBucketSampler(
        data_train_len,
        batch_size=batch_size,
        num_buckets=10,
        ratio=0,
        shuffle=True)
    # data loaders
    dataloader_train = gluon.data.DataLoader(
        dataset=data_train,
        num_workers=1,
        batch_sampler=batch_sampler,
        batchify_fn=batchify_fn)
    if task.task_name == 'MNLI':
        data_dev_matched = task('dev_matched').transform(trans, lazy=False)
        data_dev_mismatched = task('dev_mismatched').transform(trans, lazy=False)

        dataloader_dev_matched = mx.gluon.data.DataLoader(
            data_dev_matched, batch_size=dev_batch_size,
            num_workers=1, shuffle=False, batchify_fn=batchify_fn)
        dataloader_dev_mismatched = mx.gluon.data.DataLoader(
            data_dev_mismatched, batch_size=dev_batch_size,
            num_workers=1, shuffle=False, batchify_fn=batchify_fn)
        return dataloader_train, dataloader_dev_matched, \
            dataloader_dev_mismatched, num_samples_train
    else:
        data_dev = task('dev').transform(trans, lazy=False)
        dataloader_dev = mx.gluon.data.DataLoader(
            data_dev,
            batch_size=dev_batch_size,
            num_workers=1,
            shuffle=False,
            batchify_fn=batchify_fn)
        return dataloader_train, dataloader_dev, num_samples_train
311912
def decode_transformer(encoder_output, encoder_decoder_attention_bias, targets,
                       hparams, name):
  """"""
  with tf.variable_scope(name):
    targets = common_layers.flatten4d3d(targets)

    decoder_input, decoder_self_bias = (
        transformer.transformer_prepare_decoder(targets, hparams))

    decoder_input = tf.nn.dropout(decoder_input,
                                  1.0 - hparams.layer_prepostprocess_dropout)

    decoder_output = transformer.transformer_decoder(
        decoder_input, encoder_output, decoder_self_bias,
        encoder_decoder_attention_bias, hparams)
    decoder_output = tf.expand_dims(decoder_output, axis=2)
    decoder_output_shape = common_layers.shape_list(decoder_output)
    decoder_output = tf.reshape(
        decoder_output, [decoder_output_shape[0], -1, 1, hparams.hidden_size])
    # Expand since t2t expects 4d tensors.
    return decoder_output
737285
def transformer_tall_pretrain_lm_tpu_adafactor():
  """"""
  hparams = transformer_tall_pretrain_lm()
  update_hparams_for_tpu(hparams)
  hparams.max_length = 1024
  # For multi-problem on TPU we need it in absolute examples.
  hparams.batch_size = 8
  hparams.multiproblem_vocab_size = 2**16
  return hparams
57692
def _normalize(sentences):
    ''''''

    tfidf = TfidfVectorizer(tokenizer=_tokenize,
                            stop_words='english',
                            decode_error='ignore',
                            ngram_range=(1,2))
    return tfidf.fit_transform(sentences)