Local 1d self attention.
484643
def local_attention_1d(x,
                       hparams,
                       attention_type="local_unmasked",
                       q_padding="VALID",
                       kv_padding="VALID"):
  """"""
  # self-attention
  x, x_shape, is_4d = maybe_reshape_4d_to_3d(x)
  with tf.variable_scope("local_1d_self_att"):
    y = common_attention.multihead_attention(
        x,
        None,
        None,
        hparams.attention_key_channels or hparams.hidden_size,
        hparams.attention_value_channels or hparams.hidden_size,
        hparams.hidden_size,
        hparams.num_heads,
        hparams.attention_dropout,
        attention_type=attention_type,
        shared_rel=hparams.shared_rel,
        block_width=hparams.block_width,
        block_length=hparams.block_length,
        q_padding=q_padding,
        kv_padding=kv_padding,
        q_filter_width=hparams.q_filter_width,
        kv_filter_width=hparams.kv_filter_width,
        make_image_summary=False,
        name="self_attention")
    if is_4d:
      y = tf.reshape(y, x_shape)
    return y
250904
def local_attention2d_spatial_decoder(x, kv_dim, heads_dim,
                                      feedforward_dim, hparams):
  """"""
  batch_dim, length_dim, model_dim = x.shape.dims
  blocks_h_dim = mtf.Dimension("blocksh", hparams.block_height)
  blocks_w_dim = mtf.Dimension("blocksw", hparams.block_width)
  num_h_blocks_dim = mtf.Dimension("num_h_blocks",
                                   hparams.img_len // hparams.block_height)
  num_w_blocks_dim = mtf.Dimension(
      "num_w_blocks",
      hparams.img_len * hparams.num_channels // hparams.block_width)
  x = mtf.transpose(
      mtf.reshape(
          x,
          mtf.Shape([
              batch_dim, num_h_blocks_dim, blocks_h_dim,
              num_w_blocks_dim, blocks_w_dim, model_dim
          ])),
      mtf.Shape([
          batch_dim, num_h_blocks_dim, num_w_blocks_dim,
          blocks_h_dim, blocks_w_dim, model_dim
      ]))
  # Image Transformer Decoder
  # [ self attention - ffn - residual + dropout] x n
  for layer in range(hparams.num_decoder_layers):
    layer_name = "decoder_layer_%d" % layer
    with tf.variable_scope(layer_name):
      # Self attention layer
      x += layer_prepostprocess_dropout(
          mtf.layers.local_2d_self_attention_spatial_blocks(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_att"),
              kv_dim,
              heads_dim,
              memory_h_dim=num_h_blocks_dim,
              memory_w_dim=num_w_blocks_dim,
              name="self_att"), hparams)
      # ffn layer
      x += layer_prepostprocess_dropout(
          mtf.layers.dense_relu_dense(
              mtf.layers.layer_norm(x, model_dim, name="layer_norm_ffn"),
              feedforward_dim,
              hparams.dropout,
              dropout_broadcast_dims=[length_dim]), hparams)

  output = mtf.layers.layer_norm(x, model_dim, name="final_layer_norm")
  return output
1266936
def multihead_attention_vars(
    mesh, heads, io_channels, kv_channels,
    master_dtype, slice_dtype, activation_dtype):
  """"""
  return multihead_attention_params(
      mesh, heads, io_channels, kv_channels,
      mtf.VariableDType(master_dtype, slice_dtype, activation_dtype),
      combine=True)
1196387
function Convolution(kernels, size, count, stride, bias, margin){
	var N, K;

	if(type(bias) === "Tensor"){
		this.bias = bias;
		console.assert(this.bias.shape[0] === 1, "Bias Tensor must have first dimension of length one.");
		K = this.bias.shape[1];
	} else {
		K = bias.length;
		this.bias = new Tensor([1, K], bias);
	}


	// transpose kernels are required
	if(type(kernels) === "Tensor"){
		console.assert(this.kernels.shape[0] === K, "Kernel Tensor first dimension must equal Bias Tensor second dimension.");
		this.kernels = kernels;
	} else {
		N = (kernels.length / K) | 0;
		console.assert(K * N === kernels.length, "Kernel Array must have length equal to a multiple of bias array length.");

		this.kernels = new Tensor([K, N], kernels);
	}

	// kernel patch size
	this.size = size;
	// kernel count
	this.count = count;

	// stride between kernel patches
	this.stride = stride;
	// margin to add
	this.margin = margin;
}
1204395
def local_attention_2d(x, hparams, attention_type="local_attention_2d"):
  """"""
  # self-attention
  with tf.variable_scope("local_2d_self_att"):
    y = common_attention.multihead_attention_2d(
        x,
        None,
        hparams.attention_key_channels or hparams.hidden_size,
        hparams.attention_value_channels or hparams.hidden_size,
        hparams.hidden_size,
        hparams.num_heads,
        attention_type=attention_type,
        query_shape=hparams.query_shape,
        memory_flange=hparams.memory_flange,
        name="self_attention")
  return y
1129914
def inferObjectsWithRandomMovements(self, objectPlacements, maxTouches=20,
                                      settlingTime=2):
    

    for monitor in self.monitors.values():
      monitor.afterPlaceObjects(objectPlacements)

    for objectName, objectDict in self.objects.iteritems():
      self.reset()

      visitCounts = defaultdict(int)

      learnedObjectPlacement = self.learnedObjectPlacements[objectName]

      allocentricLocation = random.choice(objectDict.keys())
      nextTransitionSDR = np.empty(0, dtype="uint32")

      # Traverse the object until it is inferred.
      success = False

      for _ in xrange(maxTouches):
        featureName = objectDict[allocentricLocation]
        egocentricLocation = (allocentricLocation[0] +
                              objectPlacements[objectName][0],
                              allocentricLocation[1] +
                              objectPlacements[objectName][1])
        featureSDR = self.features[featureName]

        steps = ([nextTransitionSDR] +
                 [np.empty(0)]*settlingTime)
        for transitionSDR in steps:
          self.doTimestep(np.empty(0), transitionSDR, featureSDR,
                          egocentricLocation, learn=False)

        visitCounts[allocentricLocation] += 1

        # We should eventually infer the egocentric location where we originally
        # learned this location on the object.
        learnedEgocentricLocation = (
          allocentricLocation[0] + learnedObjectPlacement[0],
          allocentricLocation[1] + learnedObjectPlacement[1])

        if (set(self.objectLayer.getActiveCells()) ==
            set(self.objectRepresentations[objectName]) and

            set(self.inputLayer.getActiveCells()) ==
            set(self.inputRepresentations[(featureName,
                                           learnedEgocentricLocation)]) and

            set(self.locationLayer.getActiveCells()) ==
            set(self.locations[learnedEgocentricLocation])):
          success = True
          break
        else:
          allocentricLocation, transitionName = self._selectTransition(
            allocentricLocation, objectDict, visitCounts)
          nextTransitionSDR = self.transitions[transitionName]
587557
def mean_with_attention(x, name, num_heads=4):
  """"""
  with tf.variable_scope(name):
    shape = shape_list(x)
    m = tf.reduce_mean(x, [1, 2])
    a = layers().Dense(num_heads, name="mean_attn")(x)
    s = tf.reshape(a, [shape[0], -1, num_heads])
    s = tf.nn.softmax(s, axis=1)
    s = tf.reshape(s, shape[:-1] + [1, num_heads])
    am = tf.reduce_mean(tf.expand_dims(x, axis=-1) * s, [1, 2])
    l = tf.concat([am, tf.expand_dims(m, axis=-1)], axis=-1)
    return layers().Dense(2 * shape[-1], name="mean_attn_final")(
        tf.reshape(l, [shape[0], (num_heads+1) * shape[-1]]))
494632
function predict(face) {
      return getNet().computeFaceDescriptorAsync(face)
        .then(inputDescriptor =>
          getDescriptorsByClass().map(ds => ({
            className: ds.className,
            distance: computeMeanDistance(ds.faceDescriptors, inputDescriptor)
          }))
        )
    }
1299430
def attention_lm_small():
  
  hparams = attention_lm_base()
  hparams.num_hidden_layers = 4
  hparams.hidden_size = 512
  hparams.filter_size = 2048
  hparams.layer_prepostprocess_dropout = 0.5
  return hparams
149891
def _add_attend_to_encoder_cache(cache, attention_name, hparams, num_layers,
                                 key_channels, value_channels,
                                 vars_3d_num_heads, scope_prefix,
                                 encoder_output):
  """"""
  for layer in range(num_layers):
    layer_name = "layer_%d" % layer
    with tf.variable_scope("%sdecoder/%s/%s/multihead_attention" %
                           (scope_prefix, layer_name, attention_name)):
      k_encdec = common_attention.compute_attention_component(
          encoder_output,
          key_channels,
          name="k",
          vars_3d_num_heads=vars_3d_num_heads)
      k_encdec = common_attention.split_heads(k_encdec, hparams.num_heads)
      v_encdec = common_attention.compute_attention_component(
          encoder_output,
          value_channels,
          name="v",
          vars_3d_num_heads=vars_3d_num_heads)
      v_encdec = common_attention.split_heads(v_encdec, hparams.num_heads)
    cache[layer_name][attention_name] = {
        "k_encdec": k_encdec,
        "v_encdec": v_encdec
    }
  return cache
364802
private void associateL2R( T left , T right ) {
		// make the previous new observations into the new old ones
		ImageInfo<TD> tmp = featsLeft1;
		featsLeft1 = featsLeft0; featsLeft0 = tmp;
		tmp = featsRight1;
		featsRight1 = featsRight0; featsRight0 = tmp;

		// detect and associate features in the two images
		featsLeft1.reset();
		featsRight1.reset();

//		long time0 = System.currentTimeMillis();
		describeImage(left,featsLeft1);
		describeImage(right,featsRight1);
//		long time1 = System.currentTimeMillis();

		// detect and associate features in the current stereo pair
		for( int i = 0; i < detector.getNumberOfSets(); i++ ) {
			SetMatches matches = setMatches[i];
			matches.swap();
			matches.match2to3.reset();

			FastQueue<Point2D_F64> leftLoc = featsLeft1.location[i];
			FastQueue<Point2D_F64> rightLoc = featsRight1.location[i];

			assocL2R.setSource(leftLoc,featsLeft1.description[i]);
			assocL2R.setDestination(rightLoc, featsRight1.description[i]);
			assocL2R.associate();

			FastQueue<AssociatedIndex> found = assocL2R.getMatches();

//			removeUnassociated(leftLoc,featsLeft1.description[i],rightLoc,featsRight1.description[i],found);
			setMatches(matches.match2to3, found, leftLoc.size);
		}
//		long time2 = System.currentTimeMillis();
//		System.out.println("  desc "+(time1-time0)+" assoc "+(time2-time1));
	}
1164964
function (inData, outData, width, height) {
        convolve3x3(inData, outData, width, height,
            [
                [-1, -1, -1],
                [-1, 8, -1],
                [-1, -1, -1]
            ],
            false, true, true);
    }
1280759
def attention_lm_attention_moe_tiny():
  
  hparams = attention_lm_moe_small()
  hparams.moe_layers = ""
  hparams.attention_num_experts = 128
  hparams.filter_size = 8192
  hparams.attention_type = AttentionType.LOCAL_EXPERTS
  return hparams
406614
def denoise_v1_m15():
  """"""
  hparams = xmoe2_v1()
  # no local attention
  # TODO(noam): non-masked version of local-attention
  hparams.decoder_layers = [
      "att" if l == "local_att" else l for l in hparams.decoder_layers]
  hparams.decoder_type = "denoising"
  hparams.noising_spec_train = {"type": "mask", "prob": 0.15}
  return hparams
118796
def get_dilated_1d_attention_mask(
    num_heads, block_size,
    num_blocks, memory_size, gap_size,
    name="dilated_mask"):
  """"""
  mask = np.ones((num_heads, block_size, 2*block_size), np.bool)

  # now going over every row to do the right assignment of
  # memory blocks
  for i in range(block_size):
    visible = 2*block_size  - (block_size-i)
    # You always attend to yourself, set the mask for that
    mask[:, i, -(block_size - i)] = 0
    # Maybe num_blocks can be automatically calculated?
    for j in range(num_blocks):
      for k in range(memory_size):
        index = ((gap_size + memory_size)*j) + k
        if index >= visible:
          break
        mask[:, i, -(index + block_size - i + 1)] = 0  # Verify

  # adding a num blocks dimension
  mask = np.expand_dims(mask, axis=1)
  return tf.constant(mask, dtype=tf.int32, name=name)
524188
def blk_coverage_1d(blk, size):
    
    rem = size % blk
    maxpix = size - rem
    return maxpix, rem
1129577
def oneImageNLF(img, img2=None, signal=None):
    '''
    
    '''
    x, y, weights, signal = calcNLF(img, img2, signal)
    _, fn, _ = _evaluate(x, y, weights)
    return fn, signal