Basic language table

evals/languages.py

@@ -21,21 +21,6 @@ languages["language_name"] = languages["bcp_47"].apply(
     lambda x: Language.get(x).display_name()
 )
 
-# load script codes and names
-scripts = pd.read_csv("data/ScriptCodes.csv").rename(
-    columns={"Code": "iso15924", "English Name": "script_name"}
-)
-
-
-def population(bcp_47):
-    items = {
-        re.sub(r"^[a-z]+-", "", lang): pop
-        for lang, pop in LANGUAGE_SPEAKING_POPULATION.items()
-        if re.match(rf"^{bcp_47}-[A-Z]{{2}}$", lang)
-    }
-    return items
-
-
 glottolog = pd.read_csv(
     "data/glottolog_languoid.csv/languoid.csv", na_values=[""], keep_default_na=False
 )  # Min _Nan_ Chinese is not N/A!
@@ -43,7 +28,6 @@ glottolog["bcp_47"] = glottolog["iso639P3code"].apply(
     lambda x: standardize_tag(x, macro=True) if not pd.isna(x) else None
 )
 
-
 @cache
 def language_family(bcp_47):
     languoid = glottolog[glottolog["bcp_47"] == bcp_47].iloc[0]
@@ -52,6 +36,21 @@ def language_family(bcp_47):
     family = glottolog[glottolog["id"] == languoid["family_id"]].iloc[0]
     return family["name"]
 
+languages["family"] = languages["bcp_47"].apply(language_family)
+
+# load script codes and names
+scripts = pd.read_csv("data/ScriptCodes.csv").rename(
+    columns={"Code": "iso15924", "English Name": "script_name"}
+)
+
+
+def population(bcp_47):
+    items = {
+        re.sub(r"^[a-z]+-", "", lang): pop
+        for lang, pop in LANGUAGE_SPEAKING_POPULATION.items()
+        if re.match(rf"^{bcp_47}-[A-Z]{{2}}$", lang)
+    }
+    return items
 
 def script_name(iso15924):
     return scripts[scripts["iso15924"] == iso15924]["script_name"].values[0]

evals/main.py

@@ -95,6 +95,20 @@ def make_model_table(df):
     return df
 
 
+def make_language_table(df):
+    df["task_metric"] = df["task"] + "_" + df["metric"]
+    df = df.drop(columns=["task", "metric"])
+    task_metrics = df["task_metric"].unique()
+    df = df.pivot(index="bcp_47", columns="task_metric", values="score").fillna(0).reset_index()
+    df["average"] = df[task_metrics].mean(axis=1)
+    for row in [*task_metrics, "average"]:
+        df[row] = df[row].round(2)
+    df = pd.merge(languages, df, on="bcp_47", how="outer")
+    df = df.sort_values(by="average", ascending=False)
+    df = df[["language_name", "speakers", "family", "average", "in_benchmark", *task_metrics]]
+    return df
+
+
 async def main():
     results = await evaluate()
     results, lang_results, model_results, task_results = aggregate(results)
@@ -107,9 +121,9 @@ async def main():
     with open("results.json", "w") as f:
         json.dump(all_results, f, indent=2, ensure_ascii=False)
 
-    model_table = make_model_table(model_results)
     all_tables = {
-        "model_table": serialize(model_table),
+        "model_table": serialize(make_model_table(model_results)),
+        "language_table": serialize(make_language_table(lang_results)),
     }
     with open("frontend/public/results.json", "w") as f:
         json.dump(all_tables, f, indent=2, ensure_ascii=False)

frontend/src/App.js

@@ -1,53 +1,63 @@
-import './App.css';
-import { useState, useEffect } from 'react';
-import { PrimeReactProvider } from 'primereact/api';
-import "primereact/resources/themes/lara-light-cyan/theme.css";
-import ModelTable from './components/ModelTable';
+import './App.css'
+import { useState, useEffect } from 'react'
+import { PrimeReactProvider } from 'primereact/api'
+import 'primereact/resources/themes/lara-light-cyan/theme.css'
+import ModelTable from './components/ModelTable'
+import LanguageTable from './components/LanguageTable'
 
-        
-
-function App() {
-  const [data, setData] = useState(null);
-  const [loading, setLoading] = useState(true);
-  const [error, setError] = useState(null);
+function App () {
+  const [data, setData] = useState(null)
+  const [loading, setLoading] = useState(true)
+  const [error, setError] = useState(null)
 
   useEffect(() => {
     fetch('/results.json')
       .then(response => {
         if (!response.ok) {
-          throw new Error('Network response was not ok');
+          throw new Error('Network response was not ok')
         }
-        return response.json();
+        return response.json()
       })
       .then(jsonData => {
-        setData(jsonData);
-        setLoading(false);
+        setData(jsonData)
+        setLoading(false)
       })
       .catch(err => {
-        setError(err.message);
-        setLoading(false);
-      });
-  }, []);
+        setError(err.message)
+        setLoading(false)
+      })
+  }, [])
 
   return (
-    <div className="App">
-      <header className="App-header">
-        <div className="emoji-container">
-          <span role="img" aria-label="Hugging Face Emoji" className="header-emoji">🌍</span>
+    <div className='App'>
+      <header className='App-header'>
+        <div className='emoji-container'>
+          <span
+            role='img'
+            aria-label='Hugging Face Emoji'
+            className='header-emoji'
+          >
+            🌍
+          </span>
         </div>
         <h1>Global AI Language Monitor</h1>
         <p>Tracking language proficiency of AI models for every language</p>
-        
-        <div className="data-container" style={{ width: '100%' }}>
+
+        <div className='data-container' style={{ width: '100%' }}>
           <PrimeReactProvider>
             {loading && <p>...</p>}
             {error && <p>Error: {error}</p>}
-          {data && <ModelTable data={data} />}
+            {data && (
+              <div style={{ display: 'flex', flexDirection: 'row', gap: '2rem' }}>
+                <ModelTable data={data} />
+                <LanguageTable data={data} />
+              </div>
+            )}
           </PrimeReactProvider>
         </div>
       </header>
     </div>
-  );
+  )
 }
 
-export default App;
+export default App

frontend/src/components/LanguageTable.js

@@ -0,0 +1,197 @@
+import { DataTable } from 'primereact/datatable'
+import { Column } from 'primereact/column'
+import { FilterMatchMode } from 'primereact/api'
+import { MultiSelect } from 'primereact/multiselect'
+import { useState, useEffect } from 'react'
+import { Slider } from 'primereact/slider'
+import ScoreField from './ScoreField'
+
+const LanguageTable = ({ data }) => {
+  const [filters, setFilters] = useState({
+    language_name: { value: null, matchMode: FilterMatchMode.CONTAINS },
+    family: { value: null, matchMode: FilterMatchMode.IN },
+    speakers: { value: null, matchMode: FilterMatchMode.BETWEEN },
+  })
+  const table = data.language_table
+
+  const families = [...new Set(table.map(item => item.family))]
+  const familyRowFilterTemplate = options => {
+    return (
+      <MultiSelect
+        value={options.value}
+        options={families}
+        onChange={e => {
+          options.filterApplyCallback(e.value)
+          setFilters(prevFilters => ({
+            ...prevFilters,
+            family: { value: e.value, matchMode: FilterMatchMode.IN }
+          }))
+        }}
+        placeholder='All families'
+      />
+    )
+  }
+
+  const formatPopulation = population => {
+    if (population === null) {
+      return ''
+    } else if (population < 1000) {
+      return population.toFixed(0) + ''
+    } else if (population < 1000 * 1000) {
+      return (population / 1000).toFixed(1) + 'K'
+    } else if (population < 1000 * 1000 * 1000) {
+      return (population / 1000 / 1000).toFixed(1) + 'M'
+    } else {
+      return (population / 1000 / 1000 / 1000).toFixed(1) + 'B'
+    }
+  }
+
+  const SliderWithLabel = ({ value, onChange }) => {
+    const p = 10
+    const min = 2
+    const max = 12
+    const start = value === null ? min : Math.log(value[0]) / Math.log(p)
+    const stop = value === null ? max : Math.log(value[1]) / Math.log(p)
+    const [_value, _setValue] = useState([start, stop])
+    useEffect(() => {
+      const timer = setTimeout(() => {
+        onChange({
+          value:
+            _value[0] <= min + 0.1 && _value[1] >= max - 0.1
+              ? null
+              : [p ** _value[0], p ** _value[1]]
+        })
+      }, 1000)
+      return () => clearTimeout(timer)
+    }, [_value, onChange])
+    return (
+      <div style={{ minWidth: '20rem' }}>
+        <div>{formatPopulation(p ** _value[0])}</div>
+        <div>{formatPopulation(p ** _value[1])}</div>
+        <Slider
+          value={_value}
+          onChange={e => _setValue(e.value)}
+          placeholder='All sizes'
+          min={min}
+          max={max}
+          step={0.01}
+          range
+          style={{ marginTop: '5rem' }}
+        />
+      </div>
+    )
+  }
+
+  const speakerFilterTemplate = options => {
+    return (
+      <SliderWithLabel
+        value={options.value}
+        onChange={e => {
+          options.filterApplyCallback(e.value)
+          setFilters(prevFilters => ({
+            ...prevFilters,
+            speakers: { value: e.value, matchMode: FilterMatchMode.BETWEEN }
+          }))
+        }}
+      />
+    )
+  }
+
+  const speakerBodyTemplate = rowData => {
+    const populationStr = formatPopulation(rowData.speakers)
+    return <div>{populationStr}</div>
+  }
+
+  const languageBodyTemplate = rowData => {
+    return <div style={{ fontWeight: 'bold' }}>{rowData.language_name}</div>
+  }
+
+  const scoreBodyTemplate = (field, options = {}) => {
+    const { minScore = 0, maxScore = 1 } = options
+
+    return rowData => {
+      const score = rowData[field]
+      return ScoreField(score, minScore, maxScore)
+    }
+  }
+
+  return (
+    <DataTable
+      value={table}
+      header={<>Languages</>}
+      sortField='speakers'
+      removableSort
+      filters={filters}
+      filterDisplay='menu'
+      scrollable
+      scrollHeight='500px'
+      style={{ minWidth: '200px' }}
+    >
+      <Column
+        field='language_name'
+        header='Language'
+        body={languageBodyTemplate}
+        filter
+        showFilterMatchModes={false}
+        style={{ minWidth: '5rem' }}
+        frozen
+      />
+      <Column
+        field='speakers'
+        header='Speakers'
+        body={speakerBodyTemplate}
+        filter
+        filterElement={speakerFilterTemplate}
+        showFilterMatchModes={false}
+        style={{ minWidth: '5rem' }}
+      />
+      <Column
+        field='family'
+        header='Family'
+        filter
+        showFilterMatchModes={false}
+        filterElement={familyRowFilterTemplate}
+        style={{ minWidth: '10rem' }}
+      />
+      <Column
+        field='average'
+        header='Average'
+        sortable
+        body={scoreBodyTemplate('average', { minScore: 0.4, maxScore: 0.8 })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+      <Column
+        field='translation_chrf'
+        header='Translation'
+        sortable
+        body={scoreBodyTemplate('translation_chrf', {
+          minScore: 0.4,
+          maxScore: 0.7
+        })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+      <Column
+        field='classification_accuracy'
+        header='Classification'
+        sortable
+        body={scoreBodyTemplate('classification_accuracy', {
+          minScore: 0.4,
+          maxScore: 1
+        })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+      <Column
+        field='language_modeling_chrf'
+        header='Language Modeling'
+        sortable
+        body={scoreBodyTemplate('language_modeling_chrf', {
+          minScore: 0.8,
+          maxScore: 1
+        })}
+        style={{ minWidth: '5rem', maxWidth: '10rem' }}
+      />
+    </DataTable>
+  )
+}
+
+export default LanguageTable

frontend/src/components/ModelTable.js

@@ -124,7 +124,7 @@ const ModelTable = ({ data }) => {
   }
 
   const modelBodyTemplate = rowData => {
-    return <div style={{ fontWeight: 'bold' }}>{rowData.model}</div>
+    return <div style={{ fontWeight: 'bold', height: '100%' }}>{rowData.model}</div>
   }
 
   const scoreBodyTemplate = (field, options = {}) => {
@@ -162,7 +162,7 @@ const ModelTable = ({ data }) => {
         header='Model'
         filter
         showFilterMatchModes={false}
-        style={{ minWidth: '15rem' }}
+        style={{ minWidth: '10rem' }}
         body={modelBodyTemplate}
         frozen
       />
@@ -188,7 +188,7 @@ const ModelTable = ({ data }) => {
         field='average'
         header='Average'
         sortable
-        body={scoreBodyTemplate('average', { minScore: 0.4, maxScore: 0.8 })}
+        body={scoreBodyTemplate('average', { minScore: 0.3, maxScore: 0.6 })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />
       <Column
@@ -196,7 +196,7 @@ const ModelTable = ({ data }) => {
         header='Translation'
         sortable
         body={scoreBodyTemplate('translation_chrf', {
-          minScore: 0.4,
+          minScore: 0.3,
           maxScore: 0.7
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
@@ -206,8 +206,8 @@ const ModelTable = ({ data }) => {
         header='Classification'
         sortable
         body={scoreBodyTemplate('classification_accuracy', {
-          minScore: 0.4,
-          maxScore: 1
+          minScore: 0.3,
+          maxScore: 0.8
         })}
         style={{ minWidth: '5rem', maxWidth: '10rem' }}
       />