Update app.py

app.py

@@ -262,199 +262,7 @@ def process_motion_transfer(source, prompt, mt_repaint_option, mt_repaint_image)
         import traceback
         print(f"Processing failed: {str(e)}\n{traceback.format_exc()}")
         return None, None, None, None, None
-
-def process_camera_control(source, prompt, camera_motion, tracking_method):
-    """Process camera control task"""
-    try:
-        # 保存上传的文件
-        input_media_path = save_uploaded_file(source)
-        if input_media_path is None:
-            return None, None, None
-        
-        print(f"DEBUG: Camera motion: '{camera_motion}'")
-        print(f"DEBUG: Tracking method: '{tracking_method}'")
-        
-        das = get_das_pipeline()
-        video_tensor, fps, is_video = load_media(input_media_path)
-        das.fps = fps  # 设置 das.fps 为 load_media 返回的 fps
-        
-        if not is_video:
-            tracking_method = "moge"
-            print("Image input detected, switching to MoGe")
-
-        cam_motion = CameraMotionGenerator(camera_motion)
-        repaint_img_tensor = None
-        tracking_tensor = None
-
-        if tracking_method == "moge":
-            moge = get_moge_model()
-            
-            infer_result = moge.infer(video_tensor[0].to(das.device))  # [C, H, W] in range [0,1]
-            H, W = infer_result["points"].shape[0:2]
-            pred_tracks = infer_result["points"].unsqueeze(0).repeat(49, 1, 1, 1) #[T, H, W, 3]
-            cam_motion.set_intr(infer_result["intrinsics"])
-
-            if camera_motion:
-                poses = cam_motion.get_default_motion() # shape: [49, 4, 4]
-                print("Camera motion applied")
-            else:
-                poses = torch.eye(4).unsqueeze(0).repeat(49, 1, 1)
-
-            pred_tracks_flatten = pred_tracks.reshape(video_tensor.shape[0], H*W, 3)
-            pred_tracks = cam_motion.w2s(pred_tracks_flatten, poses).reshape([video_tensor.shape[0], H, W, 3]) # [T, H, W, 3]
-
-            _, tracking_tensor = das.visualize_tracking_moge(
-                pred_tracks.cpu().numpy(), 
-                infer_result["mask"].cpu().numpy()
-            )
-            print('Export tracking video via MoGe')
-        else:
-            # 使用在CPU上运行的cotracker
-            pred_tracks, pred_visibility = generate_tracking_cotracker(video_tensor)
-            
-            # 使用封装的 VGGT 处理函数
-            extr, intr = process_vggt(video_tensor)
-            
-            cam_motion.set_intr(intr)
-            cam_motion.set_extr(extr)
-
-            if camera_motion:
-                poses = cam_motion.get_default_motion() # shape: [49, 4, 4]
-                pred_tracks_world = cam_motion.s2w_vggt(pred_tracks, extr, intr)
-                pred_tracks = cam_motion.w2s_vggt(pred_tracks_world, extr, intr, poses) # [T, N, 3]
-                print("Camera motion applied")
-            
-            tracking_path, tracking_tensor = das.visualize_tracking_cotracker(pred_tracks, pred_visibility)
-            print('Export tracking video via cotracker')
-        
-        # 返回处理结果，但不应用跟踪
-        return tracking_path, video_tensor, tracking_tensor, repaint_img_tensor, fps
-    except Exception as e:
-        import traceback
-        print(f"Processing failed: {str(e)}\n{traceback.format_exc()}")
-        return None, None, None, None, None
-
-def process_object_manipulation(source, prompt, object_motion, object_mask, tracking_method):
-    """Process object manipulation task"""
-    try:
-        # Save uploaded files
-        input_image_path = save_uploaded_file(source)
-        if input_image_path is None:
-            return None, None, None, None, None
-            
-        object_mask_path = save_uploaded_file(object_mask)
-        if object_mask_path is None:
-            print("Object mask not provided")
-            return None, None, None, None, None
         
-        das = get_das_pipeline()
-        video_tensor, fps, is_video = load_media(input_image_path)
-        das.fps = fps  # 设置 das.fps 为 load_media 返回的 fps
-        
-        if not is_video:
-            tracking_method = "moge"
-            print("Image input detected, switching to MoGe")
-
-        mask_image = Image.open(object_mask_path).convert('L')
-        mask_image = transforms.Resize((480, 720))(mask_image)
-        mask = torch.from_numpy(np.array(mask_image) > 127)
-
-        motion_generator = ObjectMotionGenerator(device=das.device)
-        repaint_img_tensor = None
-        tracking_tensor = None
-        
-        if tracking_method == "moge":
-            moge = get_moge_model()
-            
-            infer_result = moge.infer(video_tensor[0].to(das.device))  # [C, H, W] in range [0,1]
-            H, W = infer_result["points"].shape[0:2]
-            pred_tracks = infer_result["points"].unsqueeze(0).repeat(49, 1, 1, 1) #[T, H, W, 3]
-
-            pred_tracks = motion_generator.apply_motion(
-                pred_tracks=pred_tracks,
-                mask=mask,
-                motion_type=object_motion,
-                distance=50,
-                num_frames=49,
-                tracking_method="moge"
-            )
-            print(f"Object motion '{object_motion}' applied using provided mask")
-            poses = torch.eye(4).unsqueeze(0).repeat(49, 1, 1)
-            pred_tracks_flatten = pred_tracks.reshape(video_tensor.shape[0], H*W, 3)
-            
-            cam_motion = CameraMotionGenerator(None)
-            cam_motion.set_intr(infer_result["intrinsics"])
-            pred_tracks = cam_motion.w2s(pred_tracks_flatten, poses).reshape([video_tensor.shape[0], H, W, 3]) # [T, H, W, 3]
-
-            _, tracking_tensor = das.visualize_tracking_moge(
-                pred_tracks.cpu().numpy(), 
-                infer_result["mask"].cpu().numpy()
-            )
-            print('Export tracking video via MoGe')
-        else:
-            # 使用在CPU上运行的cotracker
-            pred_tracks, pred_visibility = generate_tracking_cotracker(video_tensor)
-            
-            # 使用封装的 VGGT 处理函数
-            extr, intr = process_vggt(video_tensor)
-            
-            pred_tracks = motion_generator.apply_motion(
-                pred_tracks=pred_tracks.squeeze(),
-                mask=mask,
-                motion_type=object_motion,
-                distance=50,
-                num_frames=49,
-                tracking_method="cotracker"
-            )
-            print(f"Object motion '{object_motion}' applied using provided mask")
-            
-            tracking_path, tracking_tensor = das.visualize_tracking_cotracker(pred_tracks.unsqueeze(0), pred_visibility)
-            print('Export tracking video via cotracker')
-        
-        # 返回处理结果，但不应用跟踪
-        return tracking_path, video_tensor, tracking_tensor, repaint_img_tensor, fps
-    except Exception as e:
-        import traceback
-        print(f"Processing failed: {str(e)}\n{traceback.format_exc()}")
-        return None, None, None, None, None
-
-def process_mesh_animation(source, prompt, tracking_video, ma_repaint_option, ma_repaint_image):
-    """Process mesh animation task"""
-    try:
-        # Save uploaded files
-        input_video_path = save_uploaded_file(source)
-        if input_video_path is None:
-            return None, None, None, None, None
-            
-        tracking_video_path = save_uploaded_file(tracking_video)
-        if tracking_video_path is None:
-            return None, None, None, None, None
-        
-        das = get_das_pipeline()   
-        video_tensor, fps, is_video = load_media(input_video_path)
-        das.fps = fps  # 设置 das.fps 为 load_media 返回的 fps
-        
-        tracking_tensor, tracking_fps, _ = load_media(tracking_video_path)
-        repaint_img_tensor = None
-        if ma_repaint_image is not None:
-            repaint_path = save_uploaded_file(ma_repaint_image)
-            repaint_img_tensor, _, _ = load_media(repaint_path)
-            repaint_img_tensor = repaint_img_tensor[0]  # 获取第一帧
-        elif ma_repaint_option == "Yes":
-            repainter = FirstFrameRepainter(gpu_id=GPU_ID, output_dir=OUTPUT_DIR)
-            repaint_img_tensor = repainter.repaint(
-                video_tensor[0], 
-                prompt=prompt,
-                depth_path=None
-            )
-
-        # 直接返回上传的跟踪视频路径，而不是生成新的跟踪视频
-        return tracking_video_path, video_tensor, tracking_tensor, repaint_img_tensor, fps
-    except Exception as e:
-        import traceback
-        print(f"Processing failed: {str(e)}\n{traceback.format_exc()}")
-        return None, None, None, None, None
-
 def generate_tracking_cotracker(video_tensor, density=30):
     """在CPU上生成跟踪视频，只使用第一帧的深度信息，使用矩阵运算提高效率
     
@@ -674,18 +482,6 @@ with gr.Blocks(title="Diffusion as Shader") as demo:
         apply_tracking_btn = gr.Button("Generate Video", variant="primary", size="lg", interactive=False)
         output_video = gr.Video(label="Generated Video")
 
-        examples_list = load_examples()
-        if examples_list:
-            with gr.Blocks() as examples_block:
-                gr.Examples(
-                    examples=examples_list,
-                    inputs=[source_preview, mt_repaint_preview, common_prompt, tracking_video, output_video],
-                    outputs=[source_preview, mt_repaint_preview, common_prompt, tracking_video, output_video],
-                    fn=lambda *args: args,
-                    cache_examples=True,
-                    label="Examples"
-                )
-
     with left_column:
         source_upload = gr.UploadButton("1. Upload Source", file_types=["image", "video"])
         gr.Markdown("Upload a video or image, We will extract the motion and space structure from it")
@@ -749,10 +545,22 @@ with gr.Blocks(title="Diffusion as Shader") as demo:
                 )
 
             with gr.TabItem("Camera Control"):
-                 gr.Markdown("Camera Control is not available in Huggingface Space, please deploy our GitHub project on your own machine")
+                 gr.Markdown("Camera Control is not available in Huggingface Space, please deploy our [GitHub project](https://github.com/IGL-HKUST/DiffusionAsShader) on your own machine")
 
             with gr.TabItem("Object Manipulation"):
-                 gr.Markdown("Object Manipulation is not available in Huggingface Space, please deploy our GitHub project on your own machine")
+                 gr.Markdown("Object Manipulation is not available in Huggingface Space, please deploy our [GitHub project](https://github.com/IGL-HKUST/DiffusionAsShader) on your own machine")
+
+        examples_list = load_examples()
+        if examples_list:
+            with gr.Blocks() as examples_block:
+                gr.Examples(
+                    examples=examples_list,
+                    inputs=[source_preview, mt_repaint_preview, common_prompt, tracking_video, output_video],
+                    outputs=[source_preview, mt_repaint_preview, common_prompt, tracking_video, output_video],
+                    fn=lambda *args: args,
+                    cache_examples=True,
+                    label="Examples"
+                )
     
 
 # Launch interface