Set of hyperparameters for a very small imagetransformer with MoE.
322533
def imagetransformer_moe_tiny():
  """"""
  hparams = imagetransformer_tiny()
  hparams.hidden_size = 64
  hparams.batch_size = 1
  hparams.num_hidden_layers = 3
  hparams.dec_attention_type = cia.AttentionType.MOE_LOCAL_1D
  hparams.add_hparam("moe_layers_decoder", "1")  # Which layer is MoE.
  hparams.moe_hidden_sizes = "1024"  # Hidden layer sizes (comma-separated).
  hparams.moe_num_experts = 16  # Number of experts in each MoE layer.
  hparams.moe_k = 2  # How many experts to use per batch element (try 2 or 4).
  hparams.moe_loss_coef = 1e-2  # MoE loss coefficient (1e-2 is usually ok).
  return hparams
1309399
def img2img_transformer2d_q3():
  """"""
  hparams = img2img_transformer2d_q1()
  hparams.batch_size = 2
  hparams.query_shape = (8, 16)
  hparams.memory_flange = (8, 32)
  return hparams
186160
def transformer_ae_small_noatt():
  """"""
  hparams = transformer_ae_small()
  hparams.reshape_method = "slice"
  hparams.bottleneck_kind = "dvq"
  hparams.hidden_size = 512
  hparams.num_blocks = 1
  hparams.num_decode_blocks = 1
  hparams.z_size = 12
  hparams.do_attend_decompress = False
  return hparams
84812
def imagetransformer_sep_channels_8l_8h_local_and_global_att():
  """"""
  hparams = imagetransformer_sep_channels_8l_8h()
  hparams.num_heads = 8
  hparams.batch_size = 1
  hparams.attention_key_channels = hparams.attention_value_channels = 0
  hparams.hidden_size = 256
  hparams.filter_size = 256
  hparams.num_hidden_layers = 4
  hparams.sampling_method = "random"
  hparams.local_and_global_att = True
  return hparams
1234687
def xmoe2_v1_x128():
  """"""
  hparams = xmoe2_v1()
  hparams.moe_num_experts = [16, 8]
  hparams.outer_batch_size = 8
  hparams.mesh_shape = "b0:8;b1:16"
  hparams.batch_size = 512
  hparams.learning_rate_decay_steps = 16384
  return hparams
527088
def imagetransformer_sep_channels_12l_16h_imagenet_large():
  """"""
  hparams = imagetransformer_sep_channels_8l_8h()
  hparams.num_hidden_layers = 12
  hparams.batch_size = 1
  hparams.filter_size = 2048
  hparams.num_heads = 16
  hparams.learning_rate_warmup_steps = 16000
  hparams.sampling_method = "random"
  hparams.learning_rate = 0.1
  return hparams
1037676
def transformer_moe_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.norm_type = "layer"
  hparams.hidden_size = 512
  hparams.batch_size = 4096
  hparams.max_length = 2001
  hparams.max_input_seq_length = 2000
  hparams.max_target_seq_length = 2000
  hparams.dropout = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 2000
  hparams.initializer_gain = 1.0
  hparams.num_hidden_layers = 5
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.98
  hparams.num_sampled_classes = 0
  hparams.label_smoothing = 0.0
  hparams.shared_embedding_and_softmax_weights = True
  # According to noam, ("n", "da") seems better for harder-to-learn models
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"

  # Hparams used by transformer_prepare_decoder() function
  hparams.add_hparam("pos", "timing")  # timing, none
  hparams.add_hparam("proximity_bias", False)
  hparams.add_hparam("causal_decoder_self_attention", True)

  hparams = common_attention.add_standard_attention_hparams(hparams)

  # Decoder layers type. If set, num_decoder_layers parameter will be ignored
  # and the number of decoder layer will be deduced from the string
  # See top file comment for example of usage
  hparams.add_hparam("layer_types", "")
  # Default attention type (ex: a, loc, red,...) and feed-forward type (ex: fc,
  # sep, moe,...)
  hparams.add_hparam("default_att", "a")
  hparams.add_hparam("default_ff", "fc")

  return hparams
165763
def _coarsenImage(image, f):
    '''
    
    '''
    from skimage.morphology import square
    from skimage.filters import rank
    from skimage.transform._warps import rescale
    selem = square(f)
    arri = rank.mean(image, selem=selem)
    return rescale(arri, 1 / f, order=0)
619619
def mtf_image_transformer_base_imagenet_mp128():
  """"""
  hparams = mtf_image_transformer_base_imagenet()
  hparams.mesh_shape = "model:8;batch:4"
  hparams.layout = "batch:batch;d_ff:model;heads:model"
  hparams.batch_size = 8
  hparams.img_len = 128
  hparams.block_length = 128
  hparams.num_heads = 8
  hparams.num_decoder_layers = 4
  hparams.d_ff = 4096
  hparams.learning_rate_warmup_steps = 31250
  hparams.unconditional = True
  hparams.max_length = 256*256*3
  return hparams
785699
def tenengrad(img, ksize=3):
    '''''''
    Gx = cv2.Sobel(img, ddepth=cv2.CV_64F, dx=1, dy=0, ksize=ksize)
    Gy = cv2.Sobel(img, ddepth=cv2.CV_64F, dx=0, dy=1, ksize=ksize)
    FM = Gx*Gx + Gy*Gy
    mn = cv2.mean(FM)[0]
    if np.isnan(mn):
        return np.nanmean(FM)
    return mn
66763
def mtf_bitransformer_all_layers_tiny():
  """"""
  hparams = mtf_bitransformer_tiny()
  hparams.moe_num_experts = 4
  hparams.moe_expert_x = 4
  hparams.moe_expert_y = 4
  hparams.moe_hidden_size = 512
  hparams.encoder_layers = [
      "self_att", "local_self_att", "moe_1d", "moe_2d", "drd"]
  hparams.decoder_layers = [
      "self_att", "local_self_att", "enc_att", "moe_1d", "moe_2d", "drd"]
  return hparams
195776
def _fit_strel_to_im_2d(im, strel, r, x, y):
    
    """
    elem = strel.copy()
    x_dim, y_dim = im.shape
    x_min = x-r
    x_max = x+r+1
    y_min = y-r
    y_max = y+r+1
    if x_min < 0:
        x_adj = -x_min
        elem = elem[x_adj:, :]
        x_min = 0
    elif x_max > x_dim:
        x_adj = x_max - x_dim
        elem = elem[:-x_adj, :]
    if y_min < 0:
        y_adj = -y_min
        elem = elem[:, y_adj:]
        y_min = 0
    elif y_max > y_dim:
        y_adj = y_max - y_dim
        elem = elem[:, :-y_adj]
    ex, ey = elem.shape
    im[x_min:x_min+ex, y_min:y_min+ey] += elem
    return im
356926
def imagetransformer_sep_channels_8l_8h():
  """"""
  hparams = imagetransformer_base()
  hparams.num_heads = 8
  hparams.batch_size = 1
  hparams.attention_key_channels = hparams.attention_value_channels = 0
  hparams.hidden_size = 512
  hparams.filter_size = 512
  hparams.num_hidden_layers = 8
  hparams.sampling_method = "random"
  return hparams
1113307
def imagetransformer_base_8l_8h_big_cond_dr03_dan():
  """"""
  hparams = imagetransformer_sep_channels_8l()
  hparams.block_width = 256
  hparams.block_length = 256
  hparams.hidden_size = 512
  hparams.num_heads = 8
  hparams.filter_size = 2048
  hparams.batch_size = 4
  hparams.max_length = 3075
  hparams.layer_preprocess_sequence = "none"
  hparams.layer_postprocess_sequence = "dan"
  hparams.num_decoder_layers = 8
  hparams.layer_prepostprocess_dropout = 0.3
  return hparams
520323
def img2img_transformer2d_n24():
  """"""
  hparams = img2img_transformer2d_base()
  hparams.batch_size = 1
  hparams.hidden_size = 1024
  hparams.filter_size = 2048
  hparams.layer_prepostprocess_dropout = 0.2
  hparams.num_decoder_layers = 8
  hparams.query_shape = (8, 16)
  hparams.memory_flange = (8, 32)
  return hparams
117770
def xmoe_2d():
  """"""
  hparams = xmoe_top_2()
  hparams.decoder_layers = ["att", "hmoe"] * 4
  hparams.mesh_shape = "b0:2;b1:4"
  hparams.outer_batch_size = 4
  hparams.layout = "outer_batch:b0;inner_batch:b1,expert_x:b1,expert_y:b0"
  hparams.moe_num_experts = [4, 4]
  return hparams
1146495
def normalizedGraylevelVariance(img):
    '''''''
    mean, stdev = cv2.meanStdDev(img)
    s = stdev[0]**2 / mean[0]
    return s[0]