Merge branch #HuggingFaceFW-Dev/Tasks-Explorer' into 'HuggingFaceFW/Tasks-Explorer'

app.py

@@ -7,6 +7,7 @@ import re
 from concurrent.futures import ThreadPoolExecutor
 import numpy as np
 from datetime import datetime
+from typing import Any
 
 import gradio as gr
 import pandas as pd
@@ -54,7 +55,11 @@ def fetch_repo_structure(results_uri, split_checkpoints=False, oauth_token: gr.O
         token = oauth_token.token
 
     data_folder = DataFolder(results_uri, token=token)
-    runs = [f.removeprefix("details/") for f in data_folder.list_files("details", recursive=False, include_directories=True) if f != "details"]
+    try:
+        runs = [f.removeprefix("details/") for f in data_folder.list_files("details", recursive=False, include_directories=True) if f != "details"]
+    except Exception as e:
+        print(f"Error fetching repo structure: {e}")
+        runs = []
 
     if not runs:
         return {}, gr.update(choices=[], value=None)
@@ -139,7 +144,9 @@ def fetch_run_results(results_uri, selected_run_checkpoint: list[str],
     return gr.update(choices=task_names, value=task_names[0] if task_names else None), task_runs_dict
 
 
-def render_table(df, selected_run_checkpoint: list[str], metric_names):
+def render_table(df: pd.DataFrame | None, selected_run_checkpoint: list[str], 
+                metric_names: list[str], filter_different: bool = False,
+                n_samples: int = 100):
     if df is None or not selected_run_checkpoint or not metric_names:
         return None, "0"
     
@@ -148,18 +155,24 @@ def render_table(df, selected_run_checkpoint: list[str], metric_names):
                    for metric_name in metric_names]
     other_metrics = [col for col in df.columns if col.startswith(f"metric_") and col not in kept_metrics]
     df = df.drop(columns=other_metrics)
+    
+    if filter_different:
+        df = df[df.apply(lambda row: has_different_values(row, selected_run_checkpoint, metric_names), axis=1)]
+    
     df = shorten_column_names(df, selected_run_checkpoint, metric_names)
 
-    # Sample 100
-    n_samples = len(df)
-    df = df.sample(n=min(100, len(df)), random_state=42)
+    # Get total number of samples before limiting
+    total_samples = len(df)
+    
+    # Take first n_samples instead of random sampling
+    df = df.head(n_samples)
     
     # Get column widths for better display
     column_widths = get_column_widths(df)
     return gr.Dataframe(
         value=df,
         column_widths=column_widths
-    ), str(n_samples)
+    ), str(total_samples)
 
 def update_selected_run_checkpoint(selected_runs: list[str] | None, selected_checkpoint: list[str] | None, split_checkpoints: bool):
     if not selected_runs:
@@ -301,7 +314,7 @@ def load_task_data(results_uri, selected_run_checkpoint: list[str], task_name, t
         prepared_df = pd.DataFrame({
             'prompt': df[prompt_column],
             'choices': df['choices'].apply(tuple),  # Convert lists to tuples
-            'gold': df['gold'].apply(lambda x: tuple(x) if isinstance(x, list) else x),  # Convert lists to tuples
+            'gold': df['gold'].apply(lambda x: tuple(x) if is_arary_like(x) else x),  # Convert lists to tuples
             'gold_index': df['gold_index'],
             **generative_columns,
         })
@@ -348,6 +361,34 @@ def load_task_data(results_uri, selected_run_checkpoint: list[str], task_name, t
 
     return combined_df, gr.update(choices=available_metrics, value=chosen_metrics)
 
+def has_different_values(row: pd.Series, selected_run_checkpoint: list[str], metric_names: list[str]) -> bool:
+    """Check if a row has different values across runs for any metric or generation."""
+    # Check generations
+    generation_cols = [f"generation_{run}" for run in selected_run_checkpoint]
+    generation_cols = [col for col in generation_cols if col in row.index]
+    if generation_cols:
+        generations = row[generation_cols].dropna()
+        # Convert lists to tuples for comparison and handle string values
+        unique_generations = set()
+        for gen in generations:
+            if isinstance(gen, list):
+                unique_generations.add(tuple(gen))
+            else:
+                unique_generations.add(gen)
+        if len(unique_generations) > 1:
+            return True
+    
+    # Check metrics
+    for metric in metric_names:
+        metric_cols = [f"metric_{metric}_{run}" for run in selected_run_checkpoint]
+        metric_cols = [col for col in metric_cols if col in row.index]
+        if metric_cols:
+            metrics = row[metric_cols].dropna()
+            if len(metrics.unique()) > 1:
+                return True
+    
+    return False
+
 with gr.Blocks() as demo:
     available_runs_checkpoints = gr.State({})
     results_df_full = gr.State(None)
@@ -385,6 +426,8 @@ with gr.Blocks() as demo:
             with gr.Column():
                 num_samples = gr.Text(interactive=False, label="# Samples")
                 prompt_column = gr.Radio(choices=["full_prompt", "example"], label="Prompt display", value="example")
+                filter_different = gr.Checkbox(label="Show only samples with differences", value=False)
+                n_samples_input = gr.Number(value=100, label="Number of samples to show", minimum=1, maximum=1000, step=1)
 
     # Run selection
     gr.on(
@@ -435,7 +478,7 @@ with gr.Blocks() as demo:
         outputs=[results_df_full, metric_names]
     ).then(
         fn=render_table,
-        inputs=[results_df_full, selected_run_checkpoint, metric_names],
+        inputs=[results_df_full, selected_run_checkpoint, metric_names, filter_different, n_samples_input],
         outputs=[results_df, num_samples]
     )
 
@@ -447,14 +490,14 @@ with gr.Blocks() as demo:
         outputs=[results_df_full, metric_names]
     ).then(
         fn=render_table,
-        inputs=[results_df_full, selected_run_checkpoint, metric_names],
+        inputs=[results_df_full, selected_run_checkpoint, metric_names, filter_different, n_samples_input],
         outputs=[results_df, num_samples]
     )
     
     gr.on(
-        triggers=[metric_names.input],
+        triggers=[metric_names.input, filter_different.change, n_samples_input.change],
         fn=render_table,
-        inputs=[results_df_full, selected_run_checkpoint, metric_names],
+        inputs=[results_df_full, selected_run_checkpoint, metric_names, filter_different, n_samples_input],
         outputs=[results_df, num_samples]
     )