Preprocess from frames using face detector
187182
def process_frames_face(self, frames):
        
        detector = dlib.get_frontal_face_detector()
        predictor = dlib.shape_predictor(self.face_predictor_path)
        mouth_frames = self.get_frames_mouth(detector, predictor, frames)
        self.face = np.array(frames)
        self.mouth = np.array(mouth_frames)
        if mouth_frames[0] is not None:
            self.set_data(mouth_frames)
508264
def cv_precompute(self, mask, b):
        '''
        

        '''

        # Get current chunk and mask outliers
        m1 = self.get_masked_chunk(b)
        flux = self.fraw[m1]
        K = GetCovariance(self.kernel, self.kernel_params,
                          self.time[m1], self.fraw_err[m1])
        med = np.nanmedian(flux)

        # Now mask the validation set
        M = lambda x, axis = 0: np.delete(x, mask, axis=axis)
        m2 = M(m1)
        mK = M(M(K, axis=0), axis=1)
        f = M(flux) - med

        # Pre-compute the matrices
        A = [None for i in range(self.pld_order)]
        B = [None for i in range(self.pld_order)]
        for n in range(self.pld_order):
            # Only compute up to the current PLD order
            if self.lam_idx >= n:
                X2 = self.X(n, m2)
                X1 = self.X(n, m1)
                A[n] = np.dot(X2, X2.T)
                B[n] = np.dot(X1, X2.T)
                del X1, X2

        if self.transit_model is None:
            C = 0
        else:
            C = np.zeros((len(m2), len(m2)))
            mean_transit_model = med * \
                np.sum([tm.depth * tm(self.time[m2])
                        for tm in self.transit_model], axis=0)
            f -= mean_transit_model
            for tm in self.transit_model:
                X2 = tm(self.time[m2]).reshape(-1, 1)
                C += tm.var_depth * np.dot(X2, X2.T)
                del X2

        return A, B, C, mK, f, m1, m2
1221773
def preprocess_frame(frame):
  
  # Normalize from [0.0, 1.0] -> [-0.5, 0.5]
  frame = common_layers.convert_rgb_to_real(frame)
  frame = frame - 0.5
  frame, _ = glow_ops.uniform_binning_correction(frame)
  return frame
933035
def _preprocess(self, data):
        
        transformed_data = _copy(data)
        for name, step in self._transformers[:-1]:
            transformed_data = step.fit_transform(transformed_data)
            if type(transformed_data) != _tc.SFrame:
                raise RuntimeError("The transform function in step '%s' did not"
                    " return an SFrame (got %s instead)." % (name,
                                            type(transformed_data).__name__))
        return transformed_data
1061241
def preprocess_images(raw_color_im,
                      raw_depth_im,
                      camera_intr,
                      T_camera_world,
                      workspace_box,
                      workspace_im,
                      image_proc_config):
    
    # read params
    inpaint_rescale_factor = image_proc_config['inpaint_rescale_factor']
    cluster = image_proc_config['cluster']
    cluster_tolerance = image_proc_config['cluster_tolerance']
    min_cluster_size = image_proc_config['min_cluster_size']
    max_cluster_size = image_proc_config['max_cluster_size']

    # deproject into 3D world coordinates
    point_cloud_cam = camera_intr.deproject(raw_depth_im)
    point_cloud_cam.remove_zero_points()
    point_cloud_world = T_camera_world * point_cloud_cam

    # compute the segmask for points above the box
    seg_point_cloud_world, _ = point_cloud_world.box_mask(workspace_box)
    seg_point_cloud_cam = T_camera_world.inverse() * seg_point_cloud_world
    depth_im_seg = camera_intr.project_to_image(seg_point_cloud_cam)

    # mask out objects in the known workspace
    env_pixels = depth_im_seg.pixels_farther_than(workspace_im)
    depth_im_seg._data[env_pixels[:,0], env_pixels[:,1]] = 0

    # REMOVE NOISE
    # clip low points
    low_indices = np.where(point_cloud_world.data[2,:] < workspace_box.min_pt[2])[0] 
    point_cloud_world.data[2,low_indices] = workspace_box.min_pt[2]
            
    # clip high points
    high_indices = np.where(point_cloud_world.data[2,:] > workspace_box.max_pt[2])[0] 
    point_cloud_world.data[2,high_indices] = workspace_box.max_pt[2]

    # segment out the region in the workspace (including the table)
    workspace_point_cloud_world, valid_indices = point_cloud_world.box_mask(workspace_box)
    invalid_indices = np.setdiff1d(np.arange(point_cloud_world.num_points),
                                   valid_indices)

    if cluster:
        # create new cloud
        pcl_cloud = pcl.PointCloud(workspace_point_cloud_world.data.T.astype(np.float32))
        tree = pcl_cloud.make_kdtree()
    
        # find large clusters (likely to be real objects instead of noise)
        ec = pcl_cloud.make_EuclideanClusterExtraction()
        ec.set_ClusterTolerance(cluster_tolerance)
        ec.set_MinClusterSize(min_cluster_size)
        ec.set_MaxClusterSize(max_cluster_size)
        ec.set_SearchMethod(tree)
        cluster_indices = ec.Extract()
        num_clusters = len(cluster_indices)

        # read out all points in large clusters
        filtered_points = np.zeros([3,workspace_point_cloud_world.num_points])
        cur_i = 0
        for j, indices in enumerate(cluster_indices):
            num_points = len(indices)
            points = np.zeros([3,num_points])
    
            for i, index in enumerate(indices):
                points[0,i] = pcl_cloud[index][0]
                points[1,i] = pcl_cloud[index][1]
                points[2,i] = pcl_cloud[index][2]

            filtered_points[:,cur_i:cur_i+num_points] = points.copy()
            cur_i = cur_i + num_points

        # reconstruct the point cloud
        all_points = np.c_[filtered_points[:,:cur_i], point_cloud_world.data[:,invalid_indices]]
    else:
        all_points = point_cloud_world.data
    filtered_point_cloud_world = PointCloud(all_points,
                                            frame='world')  

    # compute the filtered depth image
    filtered_point_cloud_cam = T_camera_world.inverse() * filtered_point_cloud_world
    depth_im = camera_intr.project_to_image(filtered_point_cloud_cam)    

    # form segmask
    segmask = depth_im_seg.to_binary()
    valid_px_segmask = depth_im.invalid_pixel_mask().inverse()
    segmask = segmask.mask_binary(valid_px_segmask)
    
    # inpaint
    color_im = raw_color_im.inpaint(rescale_factor=inpaint_rescale_factor)
    depth_im = depth_im.inpaint(rescale_factor=inpaint_rescale_factor)    
    return color_im, depth_im, segmask
937143
protected Planar<GrayF32> preprocess(Planar<GrayF32> image) {
		super.preprocess(image);

		// image net is BGR color order
		imageBgr.bands[0] = imageRgb.bands[2];
		imageBgr.bands[1] = imageRgb.bands[1];
		imageBgr.bands[2] = imageRgb.bands[0];

		// image needs to be between 0 and 1
		GPixelMath.divide(imageBgr,255,imageBgr);

		// Normalize the image's statistics
		for (int band = 0; band < 3; band++) {
			DataManipulationOps.normalize(imageBgr.getBand(band),mean[band],stdev[band]);
		}

		return imageBgr;
	}
865383
def image_encoder(image_feat,
                  hparams,
                  name="image_encoder",
                  save_weights_to=None,
                  make_image_summary=True):
  """"""

  x = image_feat
  with tf.variable_scope(name):
    for layer in range(hparams.num_encoder_layers or hparams.num_hidden_layers):
      with tf.variable_scope("layer_%d" % layer):
        with tf.variable_scope("self_attention"):
          y = vqa_layers.multihead_attention(
              common_layers.layer_preprocess(x, hparams),
              None,
              None,
              hparams.attention_key_channels or hparams.image_hidden_size,
              hparams.attention_value_channels or hparams.image_hidden_size,
              hparams.image_hidden_size,
              hparams.num_heads,
              hparams.attention_dropout,
              attention_type=hparams.self_attention_type,
              save_weights_to=save_weights_to,
              max_relative_position=None,
              make_image_summary=make_image_summary,
              dropout_broadcast_dims=None,
              max_length=None,
              vars_3d=False,
              scale_otproduct=hparams.scale_dotproduct)
          utils.collect_named_outputs("norms", "image_feat_self_attention",
                                      tf.norm(y, axis=-1))
          x = common_layers.layer_postprocess(x, y, hparams)
          utils.collect_named_outputs(
              "norms", "image_feat_self_attention_zero_add",
              tf.norm(x, axis=-1))
        with tf.variable_scope("ffn"):
          y = common_layers.dense_relu_dense(
              common_layers.layer_preprocess(x, hparams),
              hparams.image_filter_size,
              hparams.image_hidden_size,
              dropout=hparams.relu_dropout,
              dropout_broadcast_dims=None)
          utils.collect_named_outputs("norms", "image_feat_ffn",
                                      tf.norm(y, axis=-1))
          x = common_layers.layer_postprocess(x, y, hparams)
          utils.collect_named_outputs("norms", "image_feat_ffn_zero_add",
                                      tf.norm(x, axis=-1))
    # if normalization is done in layer_preprocess, then it should also be done
    # on the output, since the output can grow very large, being the sum of
    # a whole stack of unnormalized layer outputs.
    return common_layers.layer_preprocess(x, hparams)
1372937
def _get_corners(self, image):
        """"""
        temp = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        ret, corners = cv2.findChessboardCorners(temp,
                                                 (self.rows, self.columns))
        if not ret:
            raise ChessboardNotFoundError("No chessboard could be found.")
        cv2.cornerSubPix(temp, corners, (11, 11), (-1, -1),
                         (cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS,
                          30, 0.01))
        return corners
76436
function categorizeFrame(frame) {
    var filename = frame && frame.getFileName && frame.getFileName() && Path.normalize(frame.getFileName());
    if (!filename)
        return 'core';
    else if (filename === 'tracing.js')
        return 'tracing';
    else if (!~filename.indexOf(sep))
        return 'core';
    else if (filename.indexOf(PATH_PREFIX) !== 0)
        return 'globals';
    else if (~(filename.replace(PATH_PREFIX, '')).indexOf('node_modules'))
        return 'modules';
    else
        return 'locals';
}
376655
def frame_to_latents(frame, hparams):
  """"""
  # Preprocess
  frame = preprocess_frame(frame)

  # Encode [X_t] to [z^1_t, z^2_t .. z^l_t]
  glow_vals = glow_ops.encoder_decoder(
      "codec", frame, hparams, eps=None, reverse=False)
  z_top, _, level_eps, _, _ = glow_vals
  return z_top, level_eps
598054
def processFrame(frame):
    """"""
    # The pixels are grey-scale values from 0-255 - white = 255, black = 0
    # We want to turn the image into two values, one for the left "sensor" and one for the right.
    # There are many possible ways to do this. The very simplest way would be
    # to examine the values of two pixels - one in the centre of the left half of the image,
    # and one in the centre of the right half. What are the obvious problems with this?

    # NB: The pixels are stored in a flat (1 dimensional) array, so to calculate the index of the pixels
    # at (x,y), we do:
    # index = x + (y * width)

    width = frame.width
    height = frame.height
    pixels = frame.pixels

    USE_SIMPLE_APPROACH = False  # Change this to False to use more sophisticated approach.

    if USE_SIMPLE_APPROACH:
        # Simplest approach.
        left_centre_x = int(width / 4)      # Centre of left half
        right_centre_x = int(3 * width / 4) # Centre of right half
        centre_y = int(height / 2)          # Middle row
        left_pixel_index = left_centre_x + (centre_y * width)
        right_pixel_index = right_centre_x + (centre_y * width)
        left_sensor = float(pixels[left_pixel_index]) / 255.0
        right_sensor = float(pixels[right_pixel_index]) / 255.0
    else:
        # Obviously, the simple approach misses a lot of data, and is very susceptible to noise.
        # A better approach would be to consider more pixels.
        # You could take the average of all the pixels - it's slower but better.
        # Or you could use the median value, which is less noisy than the mean.
        # Here we calculate both, and estimate the median using histograms:
        left_total = 0
        right_total = 0
        left_hist = [0 for i in range(256)]
        right_hist = [0 for i in range(256)]
        # Create a histogram for each half of the image:
        for y in range(height):
            for x in range(int(width/2)):
                i = pixels[x + y*width]
                left_hist[i] += 1
                left_total += float(i)/255.0
            for x in range(int(width/2), width):
                i = pixels[x + y*width]
                right_hist[i] += 1
                right_total += float(i)/255.0
        # Calculate the mean values:
        left_mean, right_mean = left_total / (width*height/2), right_total / (width*height/2)
        # Now use the histogram to estimate the median value
        left_total, right_total = 0, 0
        pixels_per_half = width * height / 2
        cut_off_value = pixels_per_half / 2
        left_cut_off_point, right_cut_off_point = 0, 0
        while (left_total < cut_off_value):
            left_total += left_hist[left_cut_off_point]
            left_cut_off_point += 1
        while(right_total < cut_off_value):
            right_total += right_hist[right_cut_off_point]
            right_cut_off_point += 1
        left_median, right_median = left_cut_off_point / 255.0, right_cut_off_point / 255.0
        
        # Use the median values:
        left_sensor, right_sensor = left_median, right_median
        # Or uncomment this line to use the mean values:
        # left_sensor, right_sensor = left_mean, right_mean
    
    # In our gloomy arena, we never get particularly bright, so the sensor values tend to be low.
    # To get more action from our vehicle, we can scale up (keeping 1.0 as the max).
    # What values work well for GAIN? What happens if it's too high or too low?
    GAIN = 1.8
    left_sensor = min(1, left_sensor * GAIN)
    right_sensor = min(1, right_sensor * GAIN)
    # Done - return the values:
    return left_sensor, right_sensor
1281826
def make_preprocessing_fn(output_dir, features, keep_target):
  
  def preprocessing_fn(inputs):
    """Preprocessing function.

    Args:
      inputs: dictionary of raw input tensors

    Returns:
      A dictionary of transformed tensors
    """
    stats = json.loads(
      file_io.read_file_to_string(
          os.path.join(output_dir, STATS_FILE)).decode())

    result = {}
    for name, transform in six.iteritems(features):
      transform_name = transform['transform']
      source_column = transform['source_column']

      if transform_name == TARGET_TRANSFORM:
        if not keep_target:
          continue
        if file_io.file_exists(os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column)):
          transform_name = 'one_hot'
        else:
          transform_name = 'identity'

      if transform_name == 'identity':
        result[name] = inputs[source_column]
      elif transform_name == 'scale':
        result[name] = _scale(
            inputs[name],
            min_x_value=stats['column_stats'][source_column]['min'],
            max_x_value=stats['column_stats'][source_column]['max'],
            output_min=transform.get('value', 1) * (-1),
            output_max=transform.get('value', 1))
      elif transform_name in [ONE_HOT_TRANSFORM, MULTI_HOT_TRANSFORM]:
        vocab, ex_count = read_vocab_file(
            os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column))
        if transform_name == MULTI_HOT_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          result[name] = _string_to_int(tokens, vocab)
        else:
          result[name] = _string_to_int(inputs[source_column], vocab)
      elif transform_name == IMAGE_TRANSFORM:
        make_image_to_vec_fn = _make_image_to_vec_tito(
            name, checkpoint=transform.get('checkpoint', None))
        result[name] = make_image_to_vec_fn(inputs[source_column])
      else:
        raise ValueError('unknown transform %s' % transform_name)
    return result

  return preprocessing_fn
111450
def latents_to_frames(z_top_interp, level_eps_interp, hparams):
  """"""
  # Decode [z^1_t, z^2_t .. z^l_t] to [X_t]
  images, _, _, _ = glow_ops.encoder_decoder(
      "codec", z_top_interp, hparams, eps=level_eps_interp, reverse=True)
  images = glow_ops.postprocess(images)
  return images
231299
function detectFaces(img, faceSize = 150) {
      return locateFaces(img)
        .then(rects => getFacesFromLocations(img, rects.map(mmodRect => mmodRect.rect), faceSize))
    }
359849
def get_frames_mouth(self, detector, predictor, frames):
        
        mouth_width = 100
        mouth_height = 50
        horizontal_pad = 0.19
        normalize_ratio = None
        mouth_frames = []
        for frame in frames:
            dets = detector(frame, 1)
            shape = None
            for det in dets:
                shape = predictor(frame, det)
                i = -1
            if shape is None: # Detector doesn't detect face, just return None
                return [None]
            mouth_points = []
            for part in shape.parts():
                i += 1
                if i < 48: # Only take mouth region
                    continue
                mouth_points.append((part.x, part.y))
            np_mouth_points = np.array(mouth_points)

            mouth_centroid = np.mean(np_mouth_points[:, -2:], axis=0)

            if normalize_ratio is None:
                mouth_left = np.min(np_mouth_points[:, :-1]) * (1.0 - horizontal_pad)
                mouth_right = np.max(np_mouth_points[:, :-1]) * (1.0 + horizontal_pad)

                normalize_ratio = mouth_width / float(mouth_right - mouth_left)

            new_img_shape = (int(frame.shape[0] * normalize_ratio),
                             int(frame.shape[1] * normalize_ratio))
            resized_img = imresize(frame, new_img_shape)

            mouth_centroid_norm = mouth_centroid * normalize_ratio

            mouth_l = int(mouth_centroid_norm[0] - mouth_width / 2)
            mouth_r = int(mouth_centroid_norm[0] + mouth_width / 2)
            mouth_t = int(mouth_centroid_norm[1] - mouth_height / 2)
            mouth_b = int(mouth_centroid_norm[1] + mouth_height / 2)

            mouth_crop_image = resized_img[mouth_t:mouth_b, mouth_l:mouth_r]

            mouth_frames.append(mouth_crop_image)
        return mouth_frames
21687
def vqa_v2_preprocess_image(
    image,
    height,
    width,
    mode,
    resize_side=512,
    distort=True,
    image_model_fn="resnet_v1_152",
):
  """"""

  image = tf.image.convert_image_dtype(image, dtype=tf.float32)
  assert resize_side > 0
  if resize_side:
    image = _aspect_preserving_resize(image, resize_side)
  if mode == tf.estimator.ModeKeys.TRAIN:
    image = tf.random_crop(image, [height, width, 3])
  else:
    # Central crop, assuming resize_height > height, resize_width > width.
    image = tf.image.resize_image_with_crop_or_pad(image, height, width)

  image = tf.clip_by_value(image, 0.0, 1.0)

  if mode == tf.estimator.ModeKeys.TRAIN and distort:
    image = _flip(image)
    num_distort_cases = 4
    # pylint: disable=unnecessary-lambda
    image = _apply_with_random_selector(
        image, lambda x, ordering: _distort_color(x, ordering),
        num_cases=num_distort_cases)

  if image_model_fn.startswith("resnet_v1"):
    # resnet_v1 uses vgg preprocessing
    image = image * 255.
    image = _mean_image_subtraction(image, [_R_MEAN, _G_MEAN, _B_MEAN])
  elif image_model_fn.startswith("resnet_v2"):
    # resnet v2 uses inception preprocessing
    image = tf.subtract(image, 0.5)
    image = tf.multiply(image, 2.0)

  return image
369562
def match_via_correlation(image, template, raw_tolerance=1, normed_tolerance=0.9):
    
    h, w = image.shape
    th, tw = template.shape
    # fft based convolution enables fast matching of large images
    correlation = fftconvolve(image, template[::-1,::-1])
    # trim the returned image, fftconvolve returns an image of width: (Temp_w-1) + Im_w + (Temp_w -1), likewise height
    correlation = correlation[th-1:h, tw-1:w]
    # find images regions which are potentially matches
    match_position_dict = get_tiles_at_potential_match_regions(image, template, correlation, raw_tolerance=raw_tolerance)
    # bright spots in images can lead to false positivies- the normalisation carried out here eliminates those
    results = normalise_correlation(match_position_dict, correlation, template, normed_tolerance=normed_tolerance)
    return results