Upload folder using huggingface_hub

app/draw_diagram.py

@@ -5,17 +5,28 @@ from streamlit_echarts import st_echarts
 from streamlit.components.v1 import html
 import pandas as pd
 
-path = "./style/Leaderboard-Rename-SeaEval.csv"
-info_df = pd.read_csv(path).dropna(axis=0)
+from model_information import get_dataframe
 
-# if 'models' not in st.session_state:
-#     st.session_state.models= []
 
-def draw(folder_name, category_one, category_two, sort, num_sort):
-    
+info_df = get_dataframe()
+
+
+# path = "./style/Leaderboard-Rename-SeaEval.csv"
+# info_df = pd.read_csv(path).dropna(axis=0)
+
+#Model2Detail = {
+#    {'cross_mmlu': 'Cross-MMLU'}
+#}
+
+
+
+
+def draw(folder_name, category_one, category_two, sort, num_sort, model_size_range):
+   
     folder = f"./results/{folder_name}/"
     data_path = f'{folder}/{category_one}/{category_two}.csv'
     chart_data = pd.read_csv(data_path).dropna(axis='columns').round(3)
+
     st.markdown("""
                 <style>
                 .stMultiSelect [data-baseweb=select] span {
@@ -28,14 +39,29 @@ def draw(folder_name, category_one, category_two, sort, num_sort):
                 </style>
                 """, unsafe_allow_html=True)
     
+   
     # remap model names
     display_model_names = {key.strip() :val.strip() for key, val in zip(info_df['Original Name'], info_df['Proper Display Name'])}
+    model2sizes         = {key.strip() :val.strip() for key, val in zip(info_df['Original Name'], info_df['Model Size'])}
+
     chart_data['model_show'] = chart_data['Model'].map(display_model_names)
     chart_data['model_show'] = chart_data['model_show'].fillna(chart_data['Model'].apply(lambda x: x.replace('_', '-')))
 
+    chart_data['model_size'] = chart_data['Model'].map(model2sizes)
+    chart_data['model_size'] = chart_data['model_size'].fillna('99999')
+
+    # How to work on the model size range, filter the ones that are not in the range
+    if model_size_range != 'All':
+        if model_size_range == '<10B':
+            chart_data = chart_data[chart_data['model_size'].astype(int) < 10]
+        elif model_size_range == '10B-30B':
+            chart_data = chart_data[(chart_data['model_size'].astype(int) >= 10) & (chart_data['model_size'].astype(int) < 30)]
+        elif model_size_range == '>30B':
+            chart_data = chart_data[chart_data['model_size'].astype(int) >= 30]
+
     models = st.multiselect("Please choose the model", 
                             sorted(chart_data['model_show'].tolist()), 
-                            default=sorted(chart_data['model_show'].tolist()), 
+                            default = sorted(chart_data['model_show'].tolist()),
                             )
     
     # if 'Select All' in st.session_state.models:
@@ -43,84 +69,32 @@ def draw(folder_name, category_one, category_two, sort, num_sort):
     
     chart_data = chart_data[chart_data['model_show'].isin(models)]
 
-    if num_sort == 'Ascending':
-        ascend = True 
-    else:
-        ascend = False
-    
-    chart_data = chart_data.sort_values(by=[sort], ascending=ascend).dropna(axis=0)
-
-    if len(chart_data) == 0:
-        return
+    if len(chart_data) == 0: return
     
     min_value = round(min(chart_data.iloc[:, 1]) - 0.1*min(chart_data.iloc[:, 1]), 1) 
     max_value = round(max(chart_data.iloc[:, 1]) + 0.1*max(chart_data.iloc[:, 1]), 1)         
 
     display_names = {
-        'cross_mmlu': 'Cross-MMLU',
-        'cross_logiqa': 'Cross-LogiQA',
-        'cross_xquad': 'Cross-XQUAD',
-        'sg_eval': 'SG EVAL',
+        'cross_mmlu'        : 'Cross-MMLU',
+        'cross_logiqa'      : 'Cross-LogiQA',
+        'cross_xquad'       : 'Cross-XQUAD',
+        'sg_eval'           : 'SG EVAL',
         'sg_eval_v1_cleaned': 'SG EVAL V1 Cleaned',
-        'sg_eval_v2_mcq': 'SG EVAL V2 MCQ',
-        'sg_eval_v2_open': 'SG EVAL V2 Open Ended',
-        'us_eval': 'US EVAL', 
-        'cn_eval': 'CN EVAL', 
-        'ph_eval': 'PH EVAL'
+        'sg_eval_v2_mcq'    : 'SG EVAL V2 MCQ',
+        'sg_eval_v2_open'   : 'SG EVAL V2 Open Ended',
+        'us_eval'           : 'US EVAL',
+        'cn_eval'           : 'CN EVAL',
+        'ph_eval'           : 'PH EVAL'
     }
 
-    # breakpoint()
     data_columns = [i for i in chart_data.columns if i not in ['Model', 'model_show']]
 
-    options = {
-        # "title": {"text": f"{display_names[category_two]}"},
-        "tooltip": {
-            "trigger": "axis",
-            "axisPointer": {"type": "cross", "label": {"backgroundColor": "#6a7985"}},
-            "triggerOn": 'mousemove',
-        },
-        "legend": {"data": data_columns},
-        "toolbox": {"feature": {"saveAsImage": {}}},
-        "grid": {"left": "3%", "right": "4%", "bottom": "3%", "containLabel": True},
-        "xAxis": [
-            {
-                "type": "category",
-                "boundaryGap": True,
-                "triggerEvent": True,
-                "data":  chart_data['model_show'].tolist(),
-            }
-        ],
-        "yAxis": [{"type": "value", 
-                    "min": min_value,
-                    "max": max_value, 
-                    "boundaryGap": True
-                    # "splitNumber": 10
-                    }],
-        "series": [{
-                "name": f"{col}",
-                "type": "bar",
-                "data": chart_data[f'{col}'].tolist(),
-            } for col in data_columns],
-    }
-    
-    events = {
-        "click": "function(params) { return params.value }"
-    }
-
-    value = st_echarts(options=options, events=events, height="500px")
-
     '''
     Show table
     '''
-    # st.divider()
     with st.container():
-        # st.write("")
         st.markdown('##### TABLE')
-        # custom_css = """
-                   
-        #             """
-        # st.markdown(custom_css, unsafe_allow_html=True)
-        
+
         model_link = {key.strip(): val for key, val in zip(info_df['Proper Display Name'], info_df['Link'])}
 
         chart_data['model_link'] = chart_data['model_show'].map(model_link) 
@@ -149,3 +123,61 @@ def draw(folder_name, category_one, category_two, sort, num_sort):
                 use_container_width=True
             )
 
+
+
+    # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
+
+    # Initialize a session state variable for toggling the chart visibility
+    if "show_chart" not in st.session_state:
+        st.session_state.show_chart = False
+
+    # Create a button to toggle visibility
+    if st.button("Show Chart"):
+        st.session_state.show_chart = not st.session_state.show_chart
+
+    if st.session_state.show_chart:
+
+        with st.container():
+            st.markdown('##### CHART')
+
+
+            if num_sort == 'Ascending': ascend = True   
+            else: ascend = False
+            chart_data = chart_data.sort_values(by=[sort], ascending=ascend).dropna(axis=0)
+
+            options = {
+                # "title": {"text": f"{display_names[category_two]}"},
+                "tooltip": {
+                    "trigger": "axis",
+                    "axisPointer": {"type": "cross", "label": {"backgroundColor": "#6a7985"}},
+                    "triggerOn": 'mousemove',
+                },
+                "legend": {"data": data_columns},
+                "toolbox": {"feature": {"saveAsImage": {}}},
+                "grid": {"left": "3%", "right": "4%", "bottom": "3%", "containLabel": True},
+                "xAxis": [
+                    {
+                        "type": "category",
+                        "boundaryGap": True,
+                        "triggerEvent": True,
+                        "data":  chart_data['model_show'].tolist(),
+                    }
+                ],
+                "yAxis": [{"type": "value", 
+                            "min": min_value,
+                            "max": max_value, 
+                            "boundaryGap": True
+                            # "splitNumber": 10
+                            }],
+                "series": [{
+                        "name": f"{col}",
+                        "type": "bar",
+                        "data": chart_data[f'{col}'].tolist(),
+                    } for col in data_columns],
+            }
+            
+            events = {
+                "click": "function(params) { return params.value }"
+            }
+
+            value = st_echarts(options=options, events=events, height="500px")

app/pages.py

@@ -12,9 +12,12 @@ def dashboard():
             [![GitHub Repo stars](https://img.shields.io/github/stars/SeaEval/SeaEval?style=social)][gh]
             """)
 
-    seaeval_url = "https://seaeval.github.io/"
+    st.markdown("#### News")
+    st.markdown("Nov, 2024: Update layout and support comparison between models with similar model sizes.")
 
     st.divider()
+
+    seaeval_url = "https://seaeval.github.io/"
     st.markdown("#### What is [SeaEval](%s)?" % seaeval_url)
 
     with st.container():
@@ -26,7 +29,7 @@ def dashboard():
         st.markdown('''
 
                     ''')
-        st.markdown("##### A new benchmark for multilingual, multicultral foundation model evaluation consisting of 28 dataset as the core and keep expanding over time.")
+        st.markdown("##### A benchmark for multilingual, multicultral foundation model evaluation consisting of >30 dataset and we are keep expanding over time.")
         st.markdown(''':star: How models understand and reason with natural language?    
                     :balloon: Languages: English, Chinese, Malay, Spainish, Indonedian, Vietnamese, Filipino.
                     ''')
@@ -70,38 +73,45 @@ def dashboard():
                     ''')
 
 def cross_lingual_consistency():
-    st.title("Cross-Lingual Consistency")
+    st.title("Task: Cross-Lingual Consistency")
     
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = ['Cross-MMLU', 'Cross-XQUAD', 'Cross-LogiQA']
     
-    category_one_dict = {'Zero Shot': 'zero_shot', 
-                         'Few Shot': 'few_shot'}
-    category_two_dict = {'Cross-MMLU': 'cross_mmlu', 
-                         'Cross-XQUAD': 'cross_xquad', 
-                         'Cross-LogiQA': 'cross_logiqa'}
+    category_one_dict = {
+                        'Zero Shot': 'zero_shot',
+                        'Few Shot' : 'few_shot'
+                         }
+    
+    category_two_dict = {
+                        'Cross-MMLU'  : 'cross_mmlu',
+                        'Cross-XQUAD' : 'cross_xquad',
+                        'Cross-LogiQA': 'cross_logiqa'
+                         }
 
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
+    left, center, middle, _, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
+        category_two = st.selectbox('Dataset', filters_leveltwo)
     with middle:
-        sort = st.selectbox('Sort', ['Accuracy','Cross-Lingual Consistency', 'AC3',
-                                     'English', 'Chinese', 'Spanish', 'Vietnamese'])
+        model_size_range = st.selectbox('Model Size', ['All', '<10B', '10B-30B', '>30B'])
+
     with right:
-        sortby = st.selectbox('by', ['Ascending', 'Descending'])
+        sort = st.selectbox('Sort (For Chart)', ['Accuracy','Cross-Lingual Consistency', 'AC3',
+                                     'English', 'Chinese', 'Spanish', 'Vietnamese'])
+       
+    sortby = 'Ascending'
 
     if category_one or category_two or sort or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
         
-        draw('cross_lingual', category_one, category_two, sort, sortby)
-    # else:
-    #     draw('zero_shot', 'cross_mmlu', 'Accuracy', 'Descending')
+        draw('cross_lingual', category_one, category_two, sort, sortby, model_size_range)
+
 
 def cultural_reasoning():
-    st.title("Cultural Reasoning")
+    st.title("Task: Cultural Reasoning")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = [
@@ -115,33 +125,36 @@ def cultural_reasoning():
                         ]
     
     category_one_dict = {'Zero Shot': 'zero_shot', 
-                         'Few Shot': 'few_shot'}
+                         'Few Shot': 'few_shot'
+                         }
+    
     category_two_dict = {'SG EVAL': 'sg_eval',
-                         'SG EVAL V1 Cleaned': 'sg_eval_v1_cleaned',
-                         'SG EVAL V2 MCQ': 'sg_eval_v2_mcq',
+                         'SG EVAL V1 Cleaned'   : 'sg_eval_v1_cleaned',
+                         'SG EVAL V2 MCQ'       : 'sg_eval_v2_mcq',
                          'SG EVAL V2 Open Ended': 'sg_eval_v2_open',
-                         'US EVAL': 'us_eval', 
-                         'CN EVAL': 'cn_eval', 
-                         'PH EVAL': 'ph_eval'}
+                         'US EVAL'              : 'us_eval',
+                         'CN EVAL'              : 'cn_eval',
+                         'PH EVAL'              : 'ph_eval'
+                         }
 
-    left, center, _, right = st.columns([0.2, 0.2, 0.4, 0.2])
+    left, center, middle, _, right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
-    with right:
-        sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+        category_two = st.selectbox('Dataset', filters_leveltwo)
+    with middle:
+        model_size_range = st.selectbox('Model Size', ['All', '<10B', '10B-30B', '>30B'])
+
+    sortby = 'Ascending'
     
     if category_one or category_two or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
-        draw('cultural_reasoning', category_one, category_two, 'Accuracy',sortby)
-    # else:
-    #     draw_only_acc('cultural_reasoning', 'zero_shot', 'sg_eval', 'Descending')
+        draw('cultural_reasoning', category_one, category_two, 'Accuracy', sortby, model_size_range)
 
 
 def general_reasoning():
-    st.title("General Reasoning")
+    st.title("Task: General Reasoning")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = [
@@ -162,12 +175,15 @@ def general_reasoning():
 
     left, center, _, right = st.columns([0.2, 0.2, 0.4, 0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
-    with right:
-        sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+        category_two = st.selectbox('Dataset', filters_leveltwo)
     
+    # with right:
+    #     sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+    
+    sortby = 'Ascending'
+
     if category_one or category_two or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
@@ -176,7 +192,7 @@ def general_reasoning():
     #     draw_only_acc('general_reasoning', 'zero_shot', 'MMLU Full', 'Descending')
 
 def flores():
-    st.title("FLORES-Translation")
+    st.title("Task: FLORES-Translation")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = ['Indonesian to English', 
@@ -195,12 +211,14 @@ def flores():
 
     left, center, _, right = st.columns([0.2, 0.2, 0.4, 0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
-    with right:
-        sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
-    
+        category_two = st.selectbox('Dataset', filters_leveltwo)
+    # with right:
+    #     sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+
+    sortby = 'Ascending'
+
     if category_one or category_two or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
@@ -209,7 +227,7 @@ def flores():
     #     draw_flores_translation('zero_shot', 'Indonesian to English', 'Descending')
 
 def emotion():
-    st.title("Emotion")
+    st.title("Task: Emotion")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = [
@@ -224,12 +242,15 @@ def emotion():
 
     left, center, _, right = st.columns([0.2, 0.2, 0.4, 0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
-    with right:
-        sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
-    
+        category_two = st.selectbox('Dataset', filters_leveltwo)
+    # with right:
+    #    sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+
+    sortby = 'Ascending'
+
+
     if category_one or category_two or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
@@ -238,7 +259,7 @@ def emotion():
     #     draw_only_acc('emotion', 'zero_shot', 'Indonesian Emotion  Classification', 'Descending')
 
 def dialogue():
-    st.title("Dialogue")
+    st.title("Task: Dialogue")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = [
@@ -255,18 +276,21 @@ def dialogue():
 
     left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
+        category_two = st.selectbox('Dataset', filters_leveltwo)
     with middle:
         if category_two == 'DREAM':
             sort = st.selectbox('Sort', ['Accuracy'])
         else:
             sort = st.selectbox('Sort', ['Average', 'ROUGE-1', 'ROUGE-2', 'ROUGE-L'])
     
-    with right:
-        sortby = st.selectbox('by', ['Ascending', 'Descending'])
-    
+    #with right:
+    #    sortby = st.selectbox('by', ['Ascending', 'Descending'])
+
+    sortby = 'Ascending'
+
+
     if category_one or category_two or sort or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]
@@ -275,7 +299,7 @@ def dialogue():
     #     draw_dialogue('zero_shot', 'DREAM', sort[0],'Descending')
 
 def fundamental_nlp_tasks():
-    st.title("Fundamental NLP Tasks")
+    st.title("Task: Fundamental NLP Tasks")
 
     filters_levelone = ['Zero Shot', 'Few Shot']
     filters_leveltwo = ['OCNLI', 'C3', 'COLA', 'QQP', 'MNLI', 'QNLI', 'WNLI', 'RTE', 'MRPC']
@@ -294,12 +318,15 @@ def fundamental_nlp_tasks():
 
     left, center, _, right = st.columns([0.2, 0.2, 0.4, 0.2])
     with left:
-        category_one = st.selectbox('Select Zero / Few shot', filters_levelone)
+        category_one = st.selectbox('Zero or Few Shot', filters_levelone)
     with center:
-        category_two = st.selectbox('Select the sub-category', filters_leveltwo)
-    with right:
-        sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+        category_two = st.selectbox('Dataset', filters_leveltwo)
     
+    # with right:
+    #    sortby = st.selectbox('sorted by', ['Ascending', 'Descending'])
+    
+    sortby = 'Ascending'
+
     if category_one or category_two or sortby:
         category_one = category_one_dict[category_one]
         category_two = category_two_dict[category_two]