add some new model results

static/eval_results/Default/all_model_keywords_stats.json

@@ -5146,5 +5146,239 @@
                 "average_score": 0.20711160718581811
             }
         }
+    },
+    "Mammoth_VL": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.30194776127683565
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.2365295791606494
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.2993927028494267
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.3366347826116991
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2408454736444444
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.37895522991264047
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.48003508771929826
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.27232427744946475
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.24522937191710698
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.11457024299726488
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.18941525254390731
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.1718334741390191
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.28108187023954245
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3391119999611432
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.36434285930327387
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.36915384448504296
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.15940750469262005
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.2456942956200745
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.21586513216389874
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.29359048024032264
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.2646677074112521
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.34733130661096645
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.3286125236284589
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.16358654572940287
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.25463059203015115
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.2919119209789575
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3391119999611432
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.20016011839130254
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2679179451692527
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.23600902063965679
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.15326915093278803
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.20668466311255687
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.33348955971237954
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3759170425350556
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.23894961766260706
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.351703435685048
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.26074348700688493
+            }
+        }
     }
 }

static/eval_results/Default/all_summary.json

@@ -504,5 +504,22 @@
             "micro_mean_score": 0.2704213241616509
         },
         "overall_score": 0.17379673035120966
+    },
+    "Mammoth_VL": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "macro_mean_score": 0.264052880412689,
+            "micro_mean_score": 0.2626894374387823
+        },
+        "core_cot": null,
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.37992668750165337,
+            "micro_mean_score": 0.40120378331900275
+        },
+        "overall_score": 0.27896733083008046
     }
 }

static/eval_results/SI/all_model_keywords_stats.json

@@ -1025,7 +1025,7 @@
                 "count": 82,
                 "num_samples": 1321,
                 "tasks": [],
-                "average_score": 0.5428885848330401
+                "average_score": 0.5437015929631214
             },
             "Planning": {
                 "count": 44,
@@ -1063,7 +1063,7 @@
                 "count": 83,
                 "num_samples": 1315,
                 "tasks": [],
-                "average_score": 0.5561844967140265
+                "average_score": 0.5569877095654321
             },
             "Text-Based Images and Documents": {
                 "count": 53,
@@ -1083,7 +1083,7 @@
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.46322170353087255
+                "average_score": 0.46343334374251277
             }
         },
         "output_format": {
@@ -1091,7 +1091,7 @@
                 "count": 63,
                 "num_samples": 975,
                 "tasks": [],
-                "average_score": 0.381768248331173
+                "average_score": 0.38282644938937405
             },
             "exact_text": {
                 "count": 57,
@@ -1147,7 +1147,7 @@
                 "count": 102,
                 "num_samples": 1713,
                 "tasks": [],
-                "average_score": 0.5451973412590135
+                "average_score": 0.5458509360302554
             },
             "Mathematical and Logical Reasoning": {
                 "count": 91,
@@ -1183,7 +1183,7 @@
                 "count": 101,
                 "num_samples": 1687,
                 "tasks": [],
-                "average_score": 0.5024587375994013
+                "average_score": 0.503118803606002
             }
         }
     },
@@ -1193,49 +1193,49 @@
                 "count": 16,
                 "num_samples": 244,
                 "tasks": [],
-                "average_score": 0.47501823646125113
+                "average_score": 0.47487599206349207
             },
             "Information_Extraction": {
                 "count": 41,
                 "num_samples": 644,
                 "tasks": [],
-                "average_score": 0.4328505884518674
+                "average_score": 0.45245079667466714
             },
             "Knowledge": {
                 "count": 77,
                 "num_samples": 1294,
                 "tasks": [],
-                "average_score": 0.5102257466534984
+                "average_score": 0.5086518140501541
             },
             "Mathematics": {
                 "count": 30,
                 "num_samples": 497,
                 "tasks": [],
-                "average_score": 0.33330909636235384
+                "average_score": 0.3853815223607656
             },
             "Metrics": {
                 "count": 3,
                 "num_samples": 45,
                 "tasks": [],
-                "average_score": 0.5095238095238095
+                "average_score": 0.4380952380952381
             },
             "Perception": {
                 "count": 82,
                 "num_samples": 1321,
                 "tasks": [],
-                "average_score": 0.5507427313044685
+                "average_score": 0.5468998820129136
             },
             "Planning": {
                 "count": 44,
                 "num_samples": 714,
                 "tasks": [],
-                "average_score": 0.19508720733284174
+                "average_score": 0.21148887498941377
             },
             "Science": {
                 "count": 22,
                 "num_samples": 469,
                 "tasks": [],
-                "average_score": 0.4351415236240936
+                "average_score": 0.48499051643275837
             }
         },
         "input_format": {
@@ -1243,37 +1243,37 @@
                 "count": 2,
                 "num_samples": 30,
                 "tasks": [],
-                "average_score": 0.3555116262572404
+                "average_score": 0.3348446026637953
             },
             "Artistic and Creative Content": {
                 "count": 22,
                 "num_samples": 389,
                 "tasks": [],
-                "average_score": 0.5404112582997231
+                "average_score": 0.5535202379362348
             },
             "Diagrams and Data Visualizations": {
                 "count": 88,
                 "num_samples": 1524,
                 "tasks": [],
-                "average_score": 0.42366990116355135
+                "average_score": 0.46724590271207767
             },
             "Photographs": {
                 "count": 83,
                 "num_samples": 1315,
                 "tasks": [],
-                "average_score": 0.5726107634234434
+                "average_score": 0.5613400178213946
             },
             "Text-Based Images and Documents": {
                 "count": 53,
                 "num_samples": 847,
                 "tasks": [],
-                "average_score": 0.33154206029123856
+                "average_score": 0.33052002642818507
             },
             "User Interface Screenshots": {
                 "count": 67,
                 "num_samples": 1123,
                 "tasks": [],
-                "average_score": 0.3656537691630919
+                "average_score": 0.3722082840493195
             }
         },
         "input_num": {
@@ -1281,7 +1281,7 @@
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.4427944359714585
+                "average_score": 0.45400479933257654
             }
         },
         "output_format": {
@@ -1289,37 +1289,37 @@
                 "count": 63,
                 "num_samples": 975,
                 "tasks": [],
-                "average_score": 0.3680682749954099
+                "average_score": 0.3691249729531883
             },
             "exact_text": {
                 "count": 57,
                 "num_samples": 880,
                 "tasks": [],
-                "average_score": 0.3994332512947306
+                "average_score": 0.42013434507914493
             },
             "multiple_choice": {
                 "count": 33,
                 "num_samples": 567,
                 "tasks": [],
-                "average_score": 0.5646552101097555
+                "average_score": 0.5905636451090996
             },
             "numerical_data": {
                 "count": 39,
                 "num_samples": 694,
                 "tasks": [],
-                "average_score": 0.377682596312313
+                "average_score": 0.43247267273235235
             },
             "open_ended_output": {
                 "count": 51,
                 "num_samples": 991,
                 "tasks": [],
-                "average_score": 0.5536141293443697
+                "average_score": 0.5470781816319514
             },
             "structured_output": {
                 "count": 72,
                 "num_samples": 1121,
                 "tasks": [],
-                "average_score": 0.4434262068907506
+                "average_score": 0.43823554216399857
             }
         },
         "skills": {
@@ -1327,61 +1327,61 @@
                 "count": 38,
                 "num_samples": 654,
                 "tasks": [],
-                "average_score": 0.5949207694245245
+                "average_score": 0.5955368143490581
             },
             "Domain-Specific Knowledge and Skills": {
                 "count": 46,
                 "num_samples": 897,
                 "tasks": [],
-                "average_score": 0.4385603970138852
+                "average_score": 0.4655431430975485
             },
             "Ethical and Safety Reasoning": {
                 "count": 10,
                 "num_samples": 170,
                 "tasks": [],
-                "average_score": 0.7734661654135339
+                "average_score": 0.7948947368421052
             },
             "Language Understanding and Generation": {
                 "count": 102,
                 "num_samples": 1713,
                 "tasks": [],
-                "average_score": 0.5163987806731475
+                "average_score": 0.5122400421391089
             },
             "Mathematical and Logical Reasoning": {
                 "count": 91,
                 "num_samples": 1630,
                 "tasks": [],
-                "average_score": 0.35922563291424964
+                "average_score": 0.4086167264646781
             },
             "Object Recognition and Classification": {
                 "count": 172,
                 "num_samples": 2714,
                 "tasks": [],
-                "average_score": 0.46460120838976576
+                "average_score": 0.47630441828533016
             },
             "Planning and Decision Making": {
                 "count": 23,
                 "num_samples": 356,
                 "tasks": [],
-                "average_score": 0.12518962860872068
+                "average_score": 0.09741974015331743
             },
             "Scene and Event Understanding": {
                 "count": 60,
                 "num_samples": 1004,
                 "tasks": [],
-                "average_score": 0.5961096083948861
+                "average_score": 0.5920539115535787
             },
             "Spatial and Temporal Reasoning": {
                 "count": 78,
                 "num_samples": 1273,
                 "tasks": [],
-                "average_score": 0.3571876703463106
+                "average_score": 0.3559690476405975
             },
             "Text Recognition (OCR)": {
                 "count": 101,
                 "num_samples": 1687,
                 "tasks": [],
-                "average_score": 0.4329456546880451
+                "average_score": 0.4474763430506795
             }
         }
     },
@@ -3365,6 +3365,204 @@
             }
         }
     },
+    "POINTS_15_7B": {
+        "app": {
+            "Coding": {
+                "count": 16,
+                "num_samples": 244,
+                "tasks": [],
+                "average_score": 0.31641062675070025
+            },
+            "Information_Extraction": {
+                "count": 41,
+                "num_samples": 644,
+                "tasks": [],
+                "average_score": 0.3095789895735217
+            },
+            "Knowledge": {
+                "count": 77,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.35705988992418164
+            },
+            "Mathematics": {
+                "count": 30,
+                "num_samples": 497,
+                "tasks": [],
+                "average_score": 0.24128406446063128
+            },
+            "Metrics": {
+                "count": 3,
+                "num_samples": 45,
+                "tasks": [],
+                "average_score": 0.48095238095238096
+            },
+            "Perception": {
+                "count": 82,
+                "num_samples": 1321,
+                "tasks": [],
+                "average_score": 0.4420532221275683
+            },
+            "Planning": {
+                "count": 44,
+                "num_samples": 714,
+                "tasks": [],
+                "average_score": 0.1277481304284383
+            },
+            "Science": {
+                "count": 22,
+                "num_samples": 469,
+                "tasks": [],
+                "average_score": 0.32551503611448934
+            }
+        },
+        "input_format": {
+            "3D Models and Aerial Imagery": {
+                "count": 2,
+                "num_samples": 30,
+                "tasks": [],
+                "average_score": 0.15572486552610215
+            },
+            "Artistic and Creative Content": {
+                "count": 22,
+                "num_samples": 389,
+                "tasks": [],
+                "average_score": 0.37330010041194067
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 88,
+                "num_samples": 1524,
+                "tasks": [],
+                "average_score": 0.30991539183635347
+            },
+            "Photographs": {
+                "count": 83,
+                "num_samples": 1315,
+                "tasks": [],
+                "average_score": 0.4276343385855984
+            },
+            "Text-Based Images and Documents": {
+                "count": 53,
+                "num_samples": 847,
+                "tasks": [],
+                "average_score": 0.24722440389191766
+            },
+            "User Interface Screenshots": {
+                "count": 67,
+                "num_samples": 1123,
+                "tasks": [],
+                "average_score": 0.27713077639707523
+            }
+        },
+        "input_num": {
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.32686003793394974
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 63,
+                "num_samples": 975,
+                "tasks": [],
+                "average_score": 0.3101162129247054
+            },
+            "exact_text": {
+                "count": 57,
+                "num_samples": 880,
+                "tasks": [],
+                "average_score": 0.2614010338203017
+            },
+            "multiple_choice": {
+                "count": 33,
+                "num_samples": 567,
+                "tasks": [],
+                "average_score": 0.4855568673750491
+            },
+            "numerical_data": {
+                "count": 39,
+                "num_samples": 694,
+                "tasks": [],
+                "average_score": 0.28761899055063767
+            },
+            "open_ended_output": {
+                "count": 51,
+                "num_samples": 991,
+                "tasks": [],
+                "average_score": 0.37619796536407
+            },
+            "structured_output": {
+                "count": 72,
+                "num_samples": 1121,
+                "tasks": [],
+                "average_score": 0.3069044183161335
+            }
+        },
+        "skills": {
+            "Commonsense and Social Reasoning": {
+                "count": 38,
+                "num_samples": 654,
+                "tasks": [],
+                "average_score": 0.45980379926019677
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 46,
+                "num_samples": 897,
+                "tasks": [],
+                "average_score": 0.30711751050032277
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 10,
+                "num_samples": 170,
+                "tasks": [],
+                "average_score": 0.6173496240601504
+            },
+            "Language Understanding and Generation": {
+                "count": 102,
+                "num_samples": 1713,
+                "tasks": [],
+                "average_score": 0.35317851821169477
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 91,
+                "num_samples": 1630,
+                "tasks": [],
+                "average_score": 0.28961632718794406
+            },
+            "Object Recognition and Classification": {
+                "count": 172,
+                "num_samples": 2714,
+                "tasks": [],
+                "average_score": 0.3333459246264911
+            },
+            "Planning and Decision Making": {
+                "count": 23,
+                "num_samples": 356,
+                "tasks": [],
+                "average_score": 0.08369131166291023
+            },
+            "Scene and Event Understanding": {
+                "count": 60,
+                "num_samples": 1004,
+                "tasks": [],
+                "average_score": 0.43105364189963935
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 78,
+                "num_samples": 1273,
+                "tasks": [],
+                "average_score": 0.26796963300870397
+            },
+            "Text Recognition (OCR)": {
+                "count": 101,
+                "num_samples": 1687,
+                "tasks": [],
+                "average_score": 0.3443899066327916
+            }
+        }
+    },
     "POINTS_7B": {
         "app": {
             "Coding": {
@@ -4553,6 +4751,204 @@
             }
         }
     },
+    "SmolVLM": {
+        "app": {
+            "Coding": {
+                "count": 16,
+                "num_samples": 244,
+                "tasks": [],
+                "average_score": 0.05390625
+            },
+            "Information_Extraction": {
+                "count": 41,
+                "num_samples": 644,
+                "tasks": [],
+                "average_score": 0.03906165844850793
+            },
+            "Knowledge": {
+                "count": 77,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.09639506190200878
+            },
+            "Mathematics": {
+                "count": 30,
+                "num_samples": 497,
+                "tasks": [],
+                "average_score": 0.06728619034079576
+            },
+            "Metrics": {
+                "count": 3,
+                "num_samples": 45,
+                "tasks": [],
+                "average_score": 0.2222222222222222
+            },
+            "Perception": {
+                "count": 82,
+                "num_samples": 1321,
+                "tasks": [],
+                "average_score": 0.1606753925138995
+            },
+            "Planning": {
+                "count": 44,
+                "num_samples": 714,
+                "tasks": [],
+                "average_score": 0.03272316763696074
+            },
+            "Science": {
+                "count": 22,
+                "num_samples": 469,
+                "tasks": [],
+                "average_score": 0.13950042461525716
+            }
+        },
+        "input_format": {
+            "3D Models and Aerial Imagery": {
+                "count": 2,
+                "num_samples": 30,
+                "tasks": [],
+                "average_score": 0.10013149786398344
+            },
+            "Artistic and Creative Content": {
+                "count": 22,
+                "num_samples": 389,
+                "tasks": [],
+                "average_score": 0.143657576543239
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 88,
+                "num_samples": 1524,
+                "tasks": [],
+                "average_score": 0.0979843882877799
+            },
+            "Photographs": {
+                "count": 83,
+                "num_samples": 1315,
+                "tasks": [],
+                "average_score": 0.1383108182448921
+            },
+            "Text-Based Images and Documents": {
+                "count": 53,
+                "num_samples": 847,
+                "tasks": [],
+                "average_score": 0.09044016512537822
+            },
+            "User Interface Screenshots": {
+                "count": 67,
+                "num_samples": 1123,
+                "tasks": [],
+                "average_score": 0.029842216842698305
+            }
+        },
+        "input_num": {
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.09605051124900241
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 63,
+                "num_samples": 975,
+                "tasks": [],
+                "average_score": 0.12682789970863723
+            },
+            "exact_text": {
+                "count": 57,
+                "num_samples": 880,
+                "tasks": [],
+                "average_score": 0.05128016118728194
+            },
+            "multiple_choice": {
+                "count": 33,
+                "num_samples": 567,
+                "tasks": [],
+                "average_score": 0.10496742314924135
+            },
+            "numerical_data": {
+                "count": 39,
+                "num_samples": 694,
+                "tasks": [],
+                "average_score": 0.09999979828107199
+            },
+            "open_ended_output": {
+                "count": 51,
+                "num_samples": 991,
+                "tasks": [],
+                "average_score": 0.21315705831839693
+            },
+            "structured_output": {
+                "count": 72,
+                "num_samples": 1121,
+                "tasks": [],
+                "average_score": 0.015386904208215372
+            }
+        },
+        "skills": {
+            "Commonsense and Social Reasoning": {
+                "count": 38,
+                "num_samples": 654,
+                "tasks": [],
+                "average_score": 0.1293055688222371
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 46,
+                "num_samples": 897,
+                "tasks": [],
+                "average_score": 0.077851045512787
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 10,
+                "num_samples": 170,
+                "tasks": [],
+                "average_score": 0.2222067669172932
+            },
+            "Language Understanding and Generation": {
+                "count": 102,
+                "num_samples": 1713,
+                "tasks": [],
+                "average_score": 0.12889143083611815
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 91,
+                "num_samples": 1630,
+                "tasks": [],
+                "average_score": 0.0865768026006882
+            },
+            "Object Recognition and Classification": {
+                "count": 172,
+                "num_samples": 2714,
+                "tasks": [],
+                "average_score": 0.10501451629704919
+            },
+            "Planning and Decision Making": {
+                "count": 23,
+                "num_samples": 356,
+                "tasks": [],
+                "average_score": 0.008178053830227744
+            },
+            "Scene and Event Understanding": {
+                "count": 60,
+                "num_samples": 1004,
+                "tasks": [],
+                "average_score": 0.12403047579230878
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 78,
+                "num_samples": 1273,
+                "tasks": [],
+                "average_score": 0.061765081348496016
+            },
+            "Text Recognition (OCR)": {
+                "count": 101,
+                "num_samples": 1687,
+                "tasks": [],
+                "average_score": 0.08610257462374318
+            }
+        }
+    },
     "llava_onevision_72B": {
         "app": {
             "Coding": {

static/eval_results/SI/all_summary.json

@@ -93,8 +93,8 @@
             "num_eval_tasks": 273,
             "num_eval_samples": 4116,
             "num_not_eval_samples": 0,
-            "macro_mean_score": 0.44285970964797233,
-            "micro_mean_score": 0.43756073858114675
+            "macro_mean_score": 0.4431039098921726,
+            "micro_mean_score": 0.43780369290573373
         },
         "open": {
             "num_eval_tasks": 42,
@@ -103,15 +103,15 @@
             "macro_mean_score": 0.595574663769726,
             "micro_mean_score": 0.6334563345633456
         },
-        "overall_score": 0.46322170353087283
+        "overall_score": 0.46343334374251305
     },
     "Gemini_1.5_flash_002": {
         "core": {
             "num_eval_tasks": 273,
             "num_eval_samples": 4116,
             "num_not_eval_samples": 0,
-            "macro_mean_score": 0.42188460865574384,
-            "micro_mean_score": 0.413508260447036
+            "macro_mean_score": 0.43481964330318734,
+            "micro_mean_score": 0.4297862001943635
         },
         "open": {
             "num_eval_tasks": 42,
@@ -120,7 +120,7 @@
             "macro_mean_score": 0.5787083135236054,
             "micro_mean_score": 0.6186961869618696
         },
-        "overall_score": 0.44279443597145873
+        "overall_score": 0.4540047993325765
     },
     "Gemini_1.5_pro_002": {
         "core": {
@@ -261,8 +261,8 @@
             "macro_mean_score": 0.36480000609384927,
             "micro_mean_score": 0.36205779758110807,
             "missing_tasks": [
-                "MMSoc_Misinformation_PolitiFact",
                 "table_understanding",
+                "MMSoc_Misinformation_PolitiFact",
                 "planning_screenshot_termes"
             ]
         },
@@ -316,6 +316,25 @@
         },
         "overall_score": 0.34550356262982296
     },
+    "POINTS_15_7B": {
+        "core": {
+            "num_eval_tasks": 273,
+            "num_eval_samples": 4116,
+            "num_not_eval_samples": 0,
+            "macro_mean_score": 0.31355970638319003,
+            "micro_mean_score": 0.30728203432446294,
+            "missing_tasks": []
+        },
+        "open": {
+            "num_eval_tasks": 42,
+            "num_eval_samples": 813,
+            "num_not_eval_samples": 0,
+            "macro_mean_score": 0.41331219301389166,
+            "micro_mean_score": 0.42749077490774917,
+            "missing_tasks": []
+        },
+        "overall_score": 0.32686003793395024
+    },
     "POINTS_7B": {
         "core": {
             "num_eval_tasks": 273,
@@ -430,6 +449,25 @@
         },
         "overall_score": 0.3669159632302898
     },
+    "SmolVLM": {
+        "core": {
+            "num_eval_tasks": 273,
+            "num_eval_samples": 4116,
+            "num_not_eval_samples": 0,
+            "macro_mean_score": 0.07348385181460795,
+            "micro_mean_score": 0.0732694668402814,
+            "missing_tasks": []
+        },
+        "open": {
+            "num_eval_tasks": 42,
+            "num_eval_samples": 813,
+            "num_not_eval_samples": 0,
+            "macro_mean_score": 0.2427337975725658,
+            "micro_mean_score": 0.2504920049200492,
+            "missing_tasks": []
+        },
+        "overall_score": 0.09605051124900234
+    },
     "llava_onevision_72B": {
         "core": {
             "num_eval_tasks": 273,

utils.py

@@ -29,6 +29,9 @@ MODEL_NAME_MAP = {
     "InternVL2_2B": "InternVL2-2B",
     "Molmo_7B_D": "Molmo-7B-D-0924",
     "Molmo_72B": "Molmo-72B-0924",
+    "Mammoth_VL": "Mammoth-VL-8B",
+    "SmolVLM": "SmolVLM-1.7B",
+    "POINTS_15_7B": "POINTS-1.5-8B",
 }
 
 DIMENSION_NAME_MAP = {
@@ -108,7 +111,11 @@ MODEL_URLS = {
     "Aquila_VL_2B": "https://huggingface.co/BAAI/Aquila-VL-2B-llava-qwen",
     "POINTS_7B": "https://huggingface.co/WePOINTS/POINTS-Qwen-2-5-7B-Chat",
     "Qwen2_VL_2B": "https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct",
-    "InternVL2_2B": "https://huggingface.co/OpenGVLab/InternVL2-2B"
+    "InternVL2_2B": "https://huggingface.co/OpenGVLab/InternVL2-2B",
+    "POINTS_7B": "https://huggingface.co/WePOINTS/POINTS-Qwen-2-5-7B-Chat",
+    "POINTS_15_7B": "https://huggingface.co/WePOINTS/POINTS-1-5-Qwen-2-5-7B-Chat",
+    "SmolVLM": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
+    "Mammoth_VL": "https://huggingface.co/MAmmoTH-VL/MAmmoTH-VL-8B",
 }
 
 class BaseDataLoader:
@@ -208,14 +215,20 @@ class DefaultDataLoader(BaseDataLoader):
         for model in self.MODEL_GROUPS[selected_model_group]:
             model_data = self.MODEL_DATA[model]
             summary = self.SUMMARY_DATA[model]
-            core_noncot_score = summary["core_noncot"]["macro_mean_score"]
-            core_cot_score = summary["core_cot"]["macro_mean_score"]
+            if summary["core_noncot"]:
+                core_noncot_score = summary["core_noncot"]["macro_mean_score"]
+            else:
+                core_noncot_score = '-'
+            if summary["core_cot"]:
+                core_cot_score = summary["core_cot"]["macro_mean_score"]
+            else:
+                core_cot_score = '-'
             row = {
                 "Models": get_display_model_name(model, as_link=True),
                 "Overall": round(summary["overall_score"] * 100, 2),
-                "Core w/o CoT": round(core_noncot_score * 100, 2),
-                "Core w/ CoT": round(core_cot_score * 100, 2),
-                "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2)
+                "Core w/o CoT": round(core_noncot_score * 100, 2) if core_noncot_score != '-' else '-',
+                "Core w/ CoT": round(core_cot_score * 100, 2) if core_cot_score != '-' else '-',
+                "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2) if summary["open"] else '-'
             }
             for display_name in self.SUPER_GROUPS[selected_super_group]:
                 original_keyword = self.keyword_display_map[display_name]