Update app.py

app.py

@@ -19,8 +19,11 @@ import gradio as gr
 import tempfile
 import shutil
 import subprocess
-import tensorflow as tf
+import fractions
+
+# Suppress TensorFlow warnings
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+import tensorflow as tf
 tf.get_logger().setLevel('ERROR')
 
 # Initialize models and other global variables
@@ -39,7 +42,7 @@ def frame_to_timecode(frame_num, original_fps, desired_fps):
     seconds = int(total_seconds % 60)
     milliseconds = int((total_seconds - int(total_seconds)) * 1000)
     return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{milliseconds:03d}"
-    
+
 def get_face_embedding_and_emotion(face_img):
     face_tensor = torch.tensor(face_img).permute(2, 0, 1).unsqueeze(0).float() / 255
     face_tensor = (face_tensor - 0.5) / 0.5
@@ -98,16 +101,12 @@ def extract_frames(video_path, output_folder, fps):
         print(f"FFmpeg stderr: {e.stderr}")
         raise
 
-import fractions
-
 def extract_and_align_faces_from_video(video_path, aligned_faces_folder, desired_fps):
     print(f"Processing video: {video_path}")
     
-    # Extract frames using FFmpeg
     frames_folder = os.path.join(os.path.dirname(aligned_faces_folder), 'extracted_frames')
     extract_frames(video_path, frames_folder, desired_fps)
     
-    # Get video info
     ffprobe_command = [
         'ffprobe',
         '-v', 'error',
@@ -119,7 +118,7 @@ def extract_and_align_faces_from_video(video_path, aligned_faces_folder, desired
     ]
     try:
         ffprobe_output = subprocess.check_output(ffprobe_command, universal_newlines=True).strip().split(',')
-        print(f"FFprobe output: {ffprobe_output}")  # Debugging output
+        print(f"FFprobe output: {ffprobe_output}")
         
         if len(ffprobe_output) != 2:
             raise ValueError(f"Unexpected FFprobe output format: {ffprobe_output}")
@@ -127,23 +126,20 @@ def extract_and_align_faces_from_video(video_path, aligned_faces_folder, desired
         frame_count = ffprobe_output[0]
         frame_rate = ffprobe_output[1]
         
-        print(f"Frame count (raw): {frame_count}")  # Debugging output
-        print(f"Frame rate (raw): {frame_rate}")  # Debugging output
+        print(f"Frame count (raw): {frame_count}")
+        print(f"Frame rate (raw): {frame_rate}")
         
-        # Convert frame count to int
         try:
             frame_count = int(frame_count)
         except ValueError:
             print(f"Warning: Could not convert frame count '{frame_count}' to int. Using fallback method.")
             frame_count = len([f for f in os.listdir(frames_folder) if f.endswith('.jpg')])
         
-        # Convert fractional frame rate to float
         try:
             frac = fractions.Fraction(frame_rate)
             original_fps = float(frac.numerator) / float(frac.denominator)
         except (ValueError, ZeroDivisionError):
             print(f"Warning: Could not convert frame rate '{frame_rate}' to float. Using fallback method.")
-            # Fallback: Count frames and divide by video duration
             frame_count = len([f for f in os.listdir(frames_folder) if f.endswith('.jpg')])
             duration_command = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', video_path]
             duration = float(subprocess.check_output(duration_command, universal_newlines=True).strip())
@@ -303,17 +299,27 @@ def lstm_anomaly_detection(X, feature_columns, num_anomalies=10, epochs=100, bat
     with torch.no_grad():
         reconstructed = model(X.unsqueeze(0)).squeeze(0).cpu().numpy()
 
-    mse = np.mean(np.power(X.cpu().numpy() - reconstructed, 2), axis=1)
+# Compute anomalies for all features
+    mse_all = np.mean(np.power(X.cpu().numpy() - reconstructed, 2), axis=1)
+    top_indices_all = mse_all.argsort()[-num_anomalies:][::-1]
+    anomalies_all = np.zeros(len(mse_all), dtype=bool)
+    anomalies_all[top_indices_all] = True
 
-    top_indices = mse.argsort()[-num_anomalies:][::-1]
-    anomalies = np.zeros(len(mse), dtype=bool)
-    anomalies[top_indices] = True
+    # Compute anomalies for components only
+    component_columns = [col for col in feature_columns if col.startswith('Comp')]
+    component_indices = [feature_columns.index(col) for col in component_columns]
+    mse_comp = np.mean(np.power(X.cpu().numpy()[:, component_indices] - reconstructed[:, component_indices], 2), axis=1)
+    top_indices_comp = mse_comp.argsort()[-num_anomalies:][::-1]
+    anomalies_comp = np.zeros(len(mse_comp), dtype=bool)
+    anomalies_comp[top_indices_comp] = True
 
-    return anomalies, mse, top_indices, model
+    return (anomalies_all, mse_all, top_indices_all, 
+            anomalies_comp, mse_comp, top_indices_comp, 
+            model)
 
 def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
     fig, ax = plt.subplots(figsize=(16, 8))
-    bars = ax.bar(range(len(df)), anomaly_scores, width=0.8)
+    bars = ax.bar(range(len(df)), anomaly_scores, width=0.8, color='skyblue')
     for i in top_indices:
         bars[i].set_color('red')
     ax.set_xlabel('Timecode')
@@ -325,23 +331,34 @@ def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
     plt.tight_layout()
     return fig
 
-def plot_emotion(df, emotion):
+def plot_emotion(df, emotion, num_anomalies):
     fig, ax = plt.subplots(figsize=(16, 8))
     values = df[emotion].values
-    bars = ax.bar(range(len(df)), values, width=0.8)
-    top_10_indices = np.argsort(values)[-10:]
-    for i, bar in enumerate(bars):
-        if i in top_10_indices:
-            bar.set_color('red')
+    bars = ax.bar(range(len(df)), values, width=0.8, color='lightgreen')
+    top_indices = np.argsort(values)[-num_anomalies:][::-1]
+    for i in top_indices:
+        bars[i].set_color('red')
     ax.set_xlabel('Timecode')
     ax.set_ylabel(f'{emotion.capitalize()} Score')
-    ax.set_title(f'{emotion.capitalize()} Scores Over Time')
+    ax.set_title(f'{emotion.capitalize()} Scores Over Time (Top {num_anomalies} in Red)')
     ax.xaxis.set_major_locator(MaxNLocator(nbins=100))
     ticks = ax.get_xticks()
     ax.set_xticklabels([df['Timecode'].iloc[int(tick)] if tick >= 0 and tick < len(df) else '' for tick in ticks], rotation=90, ha='right')
     plt.tight_layout()
     return fig
 
+def plot_components(df):
+    fig, ax = plt.subplots(figsize=(16, 8))
+    component_columns = [col for col in df.columns if col.startswith('Comp')]
+    for col in component_columns:
+        ax.plot(df['Time (Minutes)'], df[col], label=col)
+    ax.set_xlabel('Time (Minutes)')
+    ax.set_ylabel('Component Value')
+    ax.set_title('UMAP Components Over Time')
+    ax.legend()
+    plt.tight_layout()
+    return fig
+
 def process_video(video_path, num_anomalies, num_components, desired_fps, batch_size, progress=gr.Progress()):
     with tempfile.TemporaryDirectory() as temp_dir:
         aligned_faces_folder = os.path.join(temp_dir, 'aligned_faces')
@@ -353,10 +370,10 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
         try:
             embeddings_by_frame, emotions_by_frame, _, original_fps = extract_and_align_faces_from_video(video_path, aligned_faces_folder, desired_fps)
         except Exception as e:
-            return f"Error extracting faces: {str(e)}", None, None, None, None
+            return f"Error extracting faces: {str(e)}", None, None, None, None, None, None
 
         if not embeddings_by_frame:
-            return "No faces were extracted from the video.", None, None, None, None
+            return "No faces were extracted from the video.", None, None, None, None, None, None
 
         progress(0.3, "Clustering embeddings")
         embeddings = list(embeddings_by_frame.values())
@@ -371,25 +388,37 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
         progress(0.6, "Performing anomaly detection")
         feature_columns = [col for col in df.columns if col not in ['Frame', 'Timecode', 'Time (Minutes)', 'Embedding_Index']]
         try:
-            anomalies_all, anomaly_scores_all, top_indices_all, _ = lstm_anomaly_detection(df[feature_columns].values, feature_columns, num_anomalies=num_anomalies, batch_size=batch_size)
+            anomalies_all, anomaly_scores_all, top_indices_all, anomalies_comp, anomaly_scores_comp, top_indices_comp, _ = lstm_anomaly_detection(df[feature_columns].values, feature_columns, num_anomalies=num_anomalies, batch_size=batch_size)
         except Exception as e:
-            return f"Error in anomaly detection: {str(e)}", None, None, None, None
+            return f"Error in anomaly detection: {str(e)}", None, None, None, None, None, None
         
         progress(0.8, "Generating plots")
         try:
-            anomaly_plot = plot_anomaly_scores(df, anomaly_scores_all, top_indices_all, "All Features")
-            emotion_plots = [plot_emotion(df, emotion) for emotion in ['fear', 'sad', 'angry']]
+            anomaly_plot_all = plot_anomaly_scores(df, anomaly_scores_all, top_indices_all, "All Features")
+            anomaly_plot_comp = plot_anomaly_scores(df, anomaly_scores_comp, top_indices_comp, "Components Only")
+            components_plot = plot_components(df)
+            emotion_plots = [plot_emotion(df, emotion, num_anomalies) for emotion in ['fear', 'sad', 'angry']]
         except Exception as e:
-            return f"Error generating plots: {str(e)}", None, None, None, None
+            return f"Error generating plots: {str(e)}", None, None, None, None, None, None
 
         progress(0.9, "Preparing results")
         results = f"Top {num_anomalies} anomalies (All Features):\n"
         results += "\n".join([f"{score:.4f} at {timecode}" for score, timecode in 
                               zip(anomaly_scores_all[top_indices_all], df['Timecode'].iloc[top_indices_all].values)])
+        results += f"\n\nTop {num_anomalies} anomalies (Components Only):\n"
+        results += "\n".join([f"{score:.4f} at {timecode}" for score, timecode in 
+                              zip(anomaly_scores_comp[top_indices_comp], df['Timecode'].iloc[top_indices_comp].values)])
+
+        # Add top emotion scores to results
+        for emotion in ['fear', 'sad', 'angry']:
+            top_indices = np.argsort(df[emotion].values)[-num_anomalies:][::-1]
+            results += f"\n\nTop {num_anomalies} {emotion.capitalize()} Scores:\n"
+            results += "\n".join([f"{df[emotion].iloc[i]:.4f} at {df['Timecode'].iloc[i]}" for i in top_indices])
 
         progress(1.0, "Complete")
-        return results, anomaly_plot, emotion_plots[0], emotion_plots[1], emotion_plots[2]
-        
+        return results, anomaly_plot_all, anomaly_plot_comp, components_plot, *emotion_plots
+
+# Gradio interface
 iface = gr.Interface(
     fn=process_video,
     inputs=[
@@ -401,7 +430,9 @@ iface = gr.Interface(
     ],
     outputs=[
         gr.Textbox(label="Anomaly Detection Results"),
-        gr.Plot(label="Anomaly Scores"),
+        gr.Plot(label="Anomaly Scores (All Features)"),
+        gr.Plot(label="Anomaly Scores (Components Only)"),
+        gr.Plot(label="UMAP Components"),
         gr.Plot(label="Fear Scores"),
         gr.Plot(label="Sad Scores"),
         gr.Plot(label="Angry Scores")
@@ -412,9 +443,9 @@ iface = gr.Interface(
     It focuses on the most frequently appearing person in the video for analysis.
     
     Adjust the parameters as needed:
-    - Number of Anomalies: How many top anomalies to detect in the video
-    - Number of Components: Complexity of the facial expression model (default = 3)
-    - Desired FPS: Frames per second to analyze (lower for faster processing - optimal: 10-20)
+    - Number of Anomalies: How many top anomalies or high intensities to highlight
+    - Number of Components: Complexity of the facial expression model
+    - Desired FPS: Frames per second to analyze (lower for faster processing)
     - Batch Size: Affects processing speed and memory usage
     
     """