Repeat when connection to check significance server is timed out

server.py

@@ -20,6 +20,7 @@ from collections import namedtuple
 from xml.sax.saxutils import escape as xmlEscape, quoteattr as xmlQuoteAttr
 from threading import Lock
 
+import regex as re
 import gradio as gr
 import pandas as pd
 from huggingface_hub import HfApi, snapshot_download
@@ -72,7 +73,42 @@ def check_significance_is_reachable():
         return False
     return True
 
-def check_significance_send_task(model_a_path, model_b_path):
+REGEX_CONNECT_TIMEOUT_ERROR = re.compile(r"""ConnectTimeoutError\(.*'(.*timed out.*)'""")
+
+def get_timeout_error_msg(exception):
+    e = exception
+    
+    if isinstance(e, requests.exceptions.ConnectTimeout):
+        error_msg = REGEX_CONNECT_TIMEOUT_ERROR.search(str(e))
+        if error_msg:
+            error_msg = error_msg.group(1)
+        else:
+            error_msg = str(e).rsplit(":", 1)[-1].strip()
+    else:
+        error_msg = str(e).rsplit(":", 1)[-1].strip()
+    
+    return error_msg
+
+def check_significance_repeat_on_conn_timeout(repeat, fn, *args, **kwargs):
+    while True:
+        try:
+            result = fn(*args, **kwargs)
+        except requests.exceptions.Timeout as e:
+            error_msg = get_timeout_error_msg(e)
+            
+            if repeat:
+                print(error_msg, f"({repeat = })")
+                
+                if isinstance(repeat, int):
+                    repeat -= 1
+                
+                continue
+            else:
+                raise CheckSignificanceError(error_msg)
+        else:
+            return result
+
+def check_significance_send_task(model_a_path, model_b_path, repeat_on_conn_timeout=10):
     url = 'https://czechllm.fit.vutbr.cz/benczechmark-leaderboard/compare_significance/'
 
     # prepare and send request
@@ -84,7 +120,10 @@ def check_significance_send_task(model_a_path, model_b_path):
             'model_a': model_a_fp,
             'model_b': model_b_fp,
         }
-        response = requests.post(url, files=files, timeout=60 * 5)
+        response = check_significance_repeat_on_conn_timeout(
+            repeat_on_conn_timeout,
+            requests.post, url, files=files, timeout=60 * 5
+        )
 
     # check response
     if response.status_code == 202:
@@ -97,9 +136,12 @@ def check_significance_send_task(model_a_path, model_b_path):
 
     return result_url
 
-def check_significance_wait_for_result(result_url):
+def check_significance_wait_for_result(result_url, repeat_on_conn_timeout=10):
     while True:
-        response = requests.get(result_url, timeout=60 * 5)
+        response = check_significance_repeat_on_conn_timeout(
+            repeat_on_conn_timeout,
+            requests.get, result_url, timeout=60 * 5
+        )
         if response.status_code == 200:
             result = response.json()
             break