8

app.py

@@ -107,84 +107,53 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+                # with gr.Column():
+                #     with gr.Accordion(
+                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             finished_eval_table = gr.components.Dataframe(
+                #                 value=finished_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+                #     with gr.Accordion(
+                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             running_eval_table = gr.components.Dataframe(
+                #                 value=running_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+
+                #     with gr.Accordion(
+                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             pending_eval_table = gr.components.Dataframe(
+                #                 value=pending_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown("# ✉️✨ Submit Open model here!", elem_classes="markdown-text")
 
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
+                    model_name = gr.Textbox(label="Model name")
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    model_name
                 ],
                 submission_result,
             )

src/about.py

@@ -85,57 +85,7 @@ To reproduce our results, here is the commands you can run:
 """
 
 EVALUATION_QUEUE_TEXT = """
-<div style="font-family: 'Arial', sans-serif; padding: 20px;">
 
-    <h2 style="font-size: 28px; font-weight: bold; text-align: center; color: #4CAF50;">Model Submission</h2>
-    
-    <p style="font-size: 18px; color: #555;">
-        We are excited to invite you to submit your Vision-Language Model for evaluation. Please choose one of the following submission methods:
-    </p>
-    
-    <!-- Open-source model submission -->
-    <div style="margin-top: 20px; padding: 10px; background-color: #f1f1f1; border-radius: 8px;">
-        <h3 style="font-size: 22px; color: #6A5ACD;">1. Open-Source Model Submission</h3>
-        <p style="font-size: 16px; color: #444;">
-            If your model is open-source, simply provide the model name in the following format: <b>org/modelname</b>.
-        </p>
-        <label for="opensource-model" style="font-size: 16px; color: #444;">Model Name (org/modelname):</label>
-        <input type="text" id="opensource-model" name="opensource-model" placeholder="e.g., org/modelname" style="width: 100%; padding: 8px; margin-top: 10px; border-radius: 4px;">
-    </div>
-
-    <!-- Closed-source model submission -->
-    <div style="margin-top: 20px; padding: 10px; background-color: #f1f1f1; border-radius: 8px;">
-        <h3 style="font-size: 22px; color: #6A5ACD;">2. Closed-Source Model Submission</h3>
-        <p style="font-size: 16px; color: #444;">
-            If your model is closed-source, please select one of the following methods to submit your model:
-        </p>
-
-        <!-- Option 1: User performs evaluation -->
-        <div style="margin-top: 10px;">
-            <input type="radio" id="user-evaluates" name="evaluation-method" value="user-evaluates" style="margin-right: 8px;">
-            <label for="user-evaluates" style="font-size: 16px; color: #444;">I will evaluate my model using your provided evaluation methods and submit the model name and results.</label>
-        </div>
-
-        <!-- Option 2: We evaluate the model -->
-        <div style="margin-top: 10px;">
-            <input type="radio" id="we-evaluate" name="evaluation-method" value="we-evaluate" style="margin-right: 8px;">
-            <label for="we-evaluate" style="font-size: 16px; color: #444;">I would like you to evaluate my model. Please provide the model's URL and API key.</label>
-        </div>
-
-        <!-- User Input for Closed-Source Model -->
-        <div id="closed-source-details" style="margin-top: 20px; display: none;">
-            <p style="font-size: 16px; color: #444;">
-                For option 2, please provide the following information:
-            </p>
-            <label for="model-url" style="font-size: 16px; color: #444;">Model URL:</label>
-            <input type="url" id="model-url" name="model-url" placeholder="https://example.com" style="width: 100%; padding: 8px; margin-top: 10px; border-radius: 4px;">
-            
-            <label for="api-key" style="font-size: 16px; color: #444;">API Key:</label>
-            <input type="text" id="api-key" name="api-key" placeholder="Your API key" style="width: 100%; padding: 8px; margin-top: 10px; border-radius: 4px;">
-        </div>
-    </div>
-
-</div>
 
 """
 

src/submission/check_validity.py

@@ -88,7 +88,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(f"{info['model']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -18,11 +18,8 @@ USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    model_show: str,
+    open_source: bool
 ):
     """通过提交模型到评估队列,将信息自动保存到requests数据集中"""
     global REQUESTED_MODELS
@@ -30,46 +27,14 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
     # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
+    if open_source:
+        try:
+            model_info = API.model_info(repo_id=model)
+        except Exception:
+            return styled_error("Could not get your model information. Please fill it up properly.")
 
     modelcard_OK, error_msg = check_model_card(model)
     if not modelcard_OK:
@@ -79,29 +44,26 @@ def add_new_eval(
     print("Adding new eval")
 
     eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+        "model_name": model,
+        "model_show": model_show,
+        "open_source": open_source,
         "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
+        "submitted_time": str(current_time),
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if f"{model}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
+    if open_source:
+        user_name = model.split("/")[0] 
+        OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+        os.makedirs(OUT_DIR, exist_ok=True)
+        out_path = f"{model}_eval_request_False.json"
+    else:
+        out_path = f"{model_show}_eval_request_True.json"
+    
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))