add evaluator integration

app.py

@@ -26,6 +26,7 @@ from src.display.utils import (
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+from src.evaluator.evaluate import evaluate_model, EvaluationStatus, EvaluationResult, Tasks
 
 
 def restart_space():
@@ -87,6 +88,28 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
+# Add model evaluation functionality
+def evaluate_and_update(model_name, revision, precision, weight_type):
+    """Evaluate a model and update the leaderboard"""
+    try:
+        # Run evaluation
+        eval_result = evaluate_model(model_name, revision, precision, weight_type)
+        
+        # Add evaluation to queue
+        add_new_eval(
+            model_name=model_name,
+            revision=revision,
+            precision=precision,
+            weight_type=weight_type,
+            results=eval_result.results
+        )
+        
+        # Update leaderboard
+        LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+        return "Evaluation started successfully! Check the leaderboard for updates."
+    except Exception as e:
+        return f"Error during evaluation: {str(e)}"
+
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -98,6 +121,33 @@ with demo:
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(INTRODUCTION_TEXT)
+            gr.Markdown(LLM_BENCHMARKS_TEXT)
+            gr.Markdown(EVALUATION_QUEUE_TEXT)
+
+        with gr.TabItem("🚀 Evaluate Model", elem_id="evaluate-tab", id=3):
+            with gr.Row():
+                model_name = gr.Textbox(label="Model Name")
+                revision = gr.Textbox(label="Revision", value="main")
+            with gr.Row():
+                precision = gr.Dropdown(
+                    choices=[p.value for p in Precision],
+                    label="Precision",
+                    value="fp32"
+                )
+                weight_type = gr.Dropdown(
+                    choices=[w.value for w in WeightType],
+                    label="Weight Type",
+                    value="pytorch"
+                )
+            evaluate_button = gr.Button("Evaluate Model")
+            status_output = gr.Textbox(label="Evaluation Status")
+
+            evaluate_button.click(
+                fn=evaluate_and_update,
+                inputs=[model_name, revision, precision, weight_type],
+                outputs=[status_output]
+            )
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):

pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     "python-dateutil>=2.9.0.post0",
     "sentencepiece>=0.2.0",
     "tokenizers>=0.15.0",
+    "torch>=2.7.1",
     "tqdm>=4.67.1",
     "transformers>=4.53.1",
 ]

requirements.txt

@@ -8,6 +8,8 @@ gradio_client
 huggingface-hub>=0.18.0
 matplotlib
 numpy
+torch>=2.0.0
+scikit-learn>=1.0.0
 pandas
 python-dateutil
 tqdm

src/evaluator/evaluate.py

@@ -0,0 +1,182 @@
+import json
+import os
+from typing import Dict, Any
+from dataclasses import dataclass
+from enum import Enum
+
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from datasets import load_dataset
+
+from src.envs import API, OWNER, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
+from src.display.utils import Tasks
+
+class EvaluationStatus(Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    FINISHED = "FINISHED"
+    FAILED = "FAILED"
+
+@dataclass
+class EvaluationResult:
+    model: str
+    revision: str
+    precision: str
+    weight_type: str
+    results: Dict[str, float]
+    error: str = None
+
+def evaluate_tsac_sentiment(model, tokenizer, device):
+    """Evaluate model on TSAC sentiment analysis task"""
+    dataset = load_dataset("fbougares/tsac", split="test")
+    
+    def preprocess(examples):
+        return tokenizer(examples['text'], padding=True, truncation=True, max_length=512)
+    
+    dataset = dataset.map(preprocess, batched=True)
+    dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])
+    
+    model.eval()
+    with torch.no_grad():
+        predictions = []
+        labels = []
+        
+        for batch in dataset:
+            inputs = {k: v.to(device) for k, v in batch.items() if k != 'label'}
+            label = batch['label'].to(device)
+            
+            outputs = model(**inputs)
+            predictions.extend(outputs.logits.argmax(dim=-1).cpu().tolist())
+            labels.extend(label.cpu().tolist())
+    
+    accuracy = sum(p == l for p, l in zip(predictions, labels)) / len(predictions)
+    return accuracy
+
+def evaluate_tunisian_corpus_coverage(model, tokenizer):
+    """Evaluate model's coverage on Tunisian Dialect Corpus"""
+    dataset = load_dataset("arbml/Tunisian_Dialect_Corpus", split="test")
+    
+    def preprocess(examples):
+        return tokenizer(examples['text'], padding=True, truncation=True, max_length=512)
+    
+    dataset = dataset.map(preprocess, batched=True)
+    
+    # Calculate coverage based on tokenization
+    total_tokens = 0
+    covered_tokens = 0
+    
+    for example in dataset:
+        tokens = tokenizer.tokenize(example['text'])
+        total_tokens += len(tokens)
+        covered_tokens += len([t for t in tokens if t != tokenizer.unk_token])
+    
+    coverage = covered_tokens / total_tokens if total_tokens > 0 else 0
+    return coverage
+
+def evaluate_model(model_name: str, revision: str, precision: str, weight_type: str) -> EvaluationResult:
+    """Evaluate a single model on all tasks"""
+    try:
+        # Load model and tokenizer
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        model = AutoModelForSequenceClassification.from_pretrained(
+            model_name,
+            revision=revision,
+            torch_dtype=getattr(torch, precision),
+            trust_remote_code=True
+        ).to(device)
+        
+        tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
+        
+        # Run evaluations
+        results = {}
+        
+        # TSAC Sentiment
+        tsac_result = evaluate_tsac_sentiment(model, tokenizer, device)
+        results[Tasks.tsac_sentiment.value.benchmark] = tsac_result
+        
+        # Tunisian Corpus Coverage
+        corpus_result = evaluate_tunisian_corpus_coverage(model, tokenizer)
+        results[Tasks.tunisian_corpus.value.benchmark] = corpus_result
+        
+        return EvaluationResult(
+            model=model_name,
+            revision=revision,
+            precision=precision,
+            weight_type=weight_type,
+            results=results
+        )
+        
+    except Exception as e:
+        return EvaluationResult(
+            model=model_name,
+            revision=revision,
+            precision=precision,
+            weight_type=weight_type,
+            results={},
+            error=str(e)
+        )
+
+def process_evaluation_queue():
+    """Process all pending evaluations in the queue"""
+    # Get all pending evaluations
+    queue_dir = os.path.join(EVAL_REQUESTS_PATH)
+    pending_files = [f for f in os.listdir(queue_dir) if f.endswith('.json')]
+    
+    for file in pending_files:
+        file_path = os.path.join(queue_dir, file)
+        with open(file_path, 'r') as f:
+            eval_request = json.load(f)
+            
+        if eval_request.get('status') != EvaluationStatus.PENDING.value:
+            continue
+            
+        # Mark as running
+        eval_request['status'] = EvaluationStatus.RUNNING.value
+        with open(file_path, 'w') as f:
+            json.dump(eval_request, f, indent=2)
+            
+        # Perform evaluation
+        result = evaluate_model(
+            model_name=eval_request['model'],
+            revision=eval_request['revision'],
+            precision=eval_request['precision'],
+            weight_type=eval_request['weight_type']
+        )
+        
+        # Save results
+        if result.error:
+            eval_request['status'] = EvaluationStatus.FAILED.value
+            eval_request['error'] = result.error
+        else:
+            eval_request['status'] = EvaluationStatus.FINISHED.value
+            eval_request['results'] = result.results
+            
+        with open(file_path, 'w') as f:
+            json.dump(eval_request, f, indent=2)
+            
+        # Save to results dataset
+        result_file = os.path.join(EVAL_RESULTS_PATH, f"{result.model}_{result.precision}.json")
+        with open(result_file, 'w') as f:
+            json.dump({
+                'model': result.model,
+                'revision': result.revision,
+                'precision': result.precision,
+                'weight_type': result.weight_type,
+                'results': result.results
+            }, f, indent=2)
+            
+        # Upload to Hugging Face
+        API.upload_file(
+            path_or_fileobj=result_file,
+            path_in_repo=os.path.basename(result_file),
+            repo_id=f"{OWNER}/results",
+            repo_type="dataset",
+            commit_message=f"Add evaluation results for {result.model}"
+        )
+
+def main():
+    process_evaluation_queue()
+
+if __name__ == "__main__":
+    main()

src/evaluator/run_evaluator.py

@@ -0,0 +1,27 @@
+import time
+import sys
+import signal
+from src.evaluator.evaluate import process_evaluation_queue
+
+def signal_handler(sig, frame):
+    print("\nEvaluator shutting down...")
+    sys.exit(0)
+
+def main():
+    # Register signal handler for graceful shutdown
+    signal.signal(signal.SIGINT, signal_handler)
+    
+    print("Starting evaluator service...")
+    
+    while True:
+        try:
+            process_evaluation_queue()
+            print("Evaluation queue processed. Sleeping for 5 minutes...")
+            time.sleep(300)  # Sleep for 5 minutes
+        except Exception as e:
+            print(f"Error in evaluation process: {e}")
+            print("Retrying in 5 minutes...")
+            time.sleep(300)
+
+if __name__ == "__main__":
+    main()