External check_significance() at czechllm.fit.vutbr.cz

server.py

@@ -4,6 +4,7 @@ import json
 import os
 import hashlib
 import time
+import requests
 from collections import namedtuple
 from xml.sax.saxutils import escape as xmlEscape, quoteattr as xmlQuoteAttr
 
@@ -11,7 +12,7 @@ import gradio as gr
 import pandas as pd
 from huggingface_hub import HfApi, snapshot_download
 
-from compare_significance import check_significance, SUPPORTED_METRICS
+from compare_significance import SUPPORTED_METRICS
 
 VISIBLE_METRICS = SUPPORTED_METRICS + ["macro_f1"]
 
@@ -42,6 +43,49 @@ MARKDOWN_SPECIAL_CHARACTERS = {
     "|": "|"
 }
 
+def check_significance_send_task(model_a_path, model_b_path):
+    url = 'https://czechllm.fit.vutbr.cz/benczechmark-leaderboard/compare_significance/'
+
+    # prepare and send request
+    with (
+        open(model_a_path, 'rb') as model_a_fp,
+        open(model_b_path, 'rb') as model_b_fp,
+    ):
+        files = {
+            'model_a': model_a_fp,
+            'model_b': model_b_fp,
+        }
+        response = requests.post(url, files=files)
+
+    # check response
+    if response.status_code == 202:
+        result_url = response.url
+        #task_id = response.json()['task_id']
+    elif response.status_code == 429:
+        raise RuntimeError('Server is too busy. Please try again later.')  # TODO: try-except do raise gr.error
+    else:
+        raise RuntimeError(f'Failed to submit task. Status code: {response.status_code}')  # TODO: try-except do raise gr.error
+
+    return result_url
+
+def check_significance_wait_for_result(result_url):
+    while True:
+        response = requests.get(result_url)
+        if response.status_code == 200:
+            result = response.json()
+            break
+        elif response.status_code == 202:
+            time.sleep(5)
+        else:
+            raise RuntimeError(f'Failed to get result. Status code: {response.status_code}')  # TODO: try-except do raise gr.error
+    
+    return result['result']
+
+def check_significance(model_a_path, model_b_path):
+    result_url = check_significance_send_task(model_a_path, model_b_path)
+    result = check_significance_wait_for_result(result_url)
+    return result
+
 class LeaderboardServer:
     def __init__(self):
         self.server_address = REPO
@@ -213,10 +257,14 @@ class LeaderboardServer:
         new_tournament[new_submission_id][new_submission_id] = {
             task: False for task in self.tasks_metadata.keys()
         }
-
+        
         for competitor_id in self.submission_ids:
-            res = check_significance(new_model_file, self.submission_id_to_file[competitor_id])
-            res_inverse = check_significance(self.submission_id_to_file[competitor_id], new_model_file)
+            res = check_significance_send_task(new_model_file, self.submission_id_to_file[competitor_id])
+            res_inverse = check_significance_send_task(self.submission_id_to_file[competitor_id], new_model_file)
+            
+            res = check_significance_wait_for_result(res)
+            res_inverse = check_significance_wait_for_result(res_inverse)
+            
             new_tournament[new_submission_id][competitor_id] = {
                 task: data["significant"] for task, data in res.items()
             }