Speed things up

evals/backend.py

@@ -32,16 +32,15 @@ async def data(request: Request):
     data = json.loads(body)
     selected_languages = data.get("selectedLanguages", {})
     df = results
-    if selected_languages:
-        df = df[df["bcp_47"].isin(l["bcp_47"] for l in selected_languages)]
-
     _, lang_results, model_results, task_results = aggregate(df)
-    print(lang_results)
     # lang_results = pd.merge(languages, lang_results, on="bcp_47", how="outer")
-    model_table = make_model_table(model_results, models)
     language_table = make_language_table(lang_results, languages)
     datasets_df = pd.read_json("data/datasets.json")
     countries = make_country_table(language_table)
+    if selected_languages:
+        # the filtering is only applied for the model table
+        df = df[df["bcp_47"].isin(lang["bcp_47"] for lang in selected_languages)]
+    model_table = make_model_table(model_results, models)
     all_tables = {
         "model_table": serialize(model_table),
         "language_table": serialize(language_table),

evals/countries.py

@@ -1,12 +1,14 @@
 import re
 import xml.etree.ElementTree as ET
 from collections import defaultdict
-
+from joblib.memory import Memory
 import pandas as pd
 from language_data.population_data import LANGUAGE_SPEAKING_POPULATION
 from language_data.util import data_filename
 
+cache = Memory(location=".cache", verbose=0).cache
 
+@cache
 def get_population_data():
     filename = data_filename("supplementalData.xml")
     root = ET.fromstring(open(filename).read())
@@ -28,7 +30,7 @@ def population(bcp_47):
     }
     return items
 
-
+@cache
 def make_country_table(language_table):
     countries = defaultdict(list)
     for lang in language_table.itertuples():

frontend/src/components/LanguageTable.js

@@ -124,7 +124,7 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
 
   return (
     <DataTable
-      value={data}
+      value={data.filter(item => !selectedLanguages.some(l => l.bcp_47 === item.bcp_47))}
       header={<>Languages</>}
       sortField='speakers'
       removableSort
@@ -134,9 +134,11 @@ const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
       selection={selectedLanguages}
       onSelectionChange={e => setSelectedLanguages(e.value)}
       frozenValue={selectedLanguages}
+      virtualScrollerOptions={{ itemSize: 50 }}
       scrollable
       scrollHeight='600px'
       id='language-table'
+      style={{ width: '800px' }}
     >
       <Column selectionMode="multiple" headerStyle={{ width: '3rem' }} />
       <Column