added bettertransformer and LLM.int8

app.py

@@ -4,8 +4,20 @@ import pandas as pd
 import plotly.express as px
 from apscheduler.schedulers.background import BackgroundScheduler
 
-from src.assets.text_content import TITLE, INTRODUCTION_TEXT, SINGLE_A100_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
-from src.utils import restart_space, load_dataset_repo, make_clickable_model, make_clickable_score, num_to_str
+from src.assets.text_content import (
+    TITLE,
+    INTRODUCTION_TEXT,
+    SINGLE_A100_TEXT,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+)
+from src.utils import (
+    restart_space,
+    load_dataset_repo,
+    make_clickable_model,
+    make_clickable_score,
+    num_to_str,
+)
 from src.assets.css_html_js import custom_css
 
 
@@ -17,13 +29,28 @@ COLUMNS_MAPPING = {
     "model": "Model 🤗",
     "backend.name": "Backend 🏭",
     "backend.torch_dtype": "Load Dtype 📥",
-    "num_parameters": "#Parameters 📏",
+    "num_parameters": "#️⃣ Parameters 📏",
+    #
     "forward.peak_memory(MB)": "Peak Memory (MB) ⬇️",
     "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
     "average": "Average Open LLM Score ⬆️",
+    #
+    "backend.bettertransformer": "BetterTransformer 🤖",
+    "backend.load_in_8bit": "LLM.int8 🗜️",
 }
-COLUMNS_DATATYPES = ["markdown", "str", "str",
-                     "number", "number", "number", "markdown"]
+COLUMNS_DATATYPES = [
+    "markdown",
+    "str",
+    "str",
+    "str",
+    #
+    "number",
+    "number",
+    "markdown",
+    #
+    "str",
+    "str",
+]
 SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
 
 
@@ -35,17 +62,14 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
         llm_perf_dataset_repo.git_pull()
 
     # load
-    bench_df = pd.read_csv(
-        f"./llm-perf-dataset/reports/{benchmark}.csv")
-    scores_df = pd.read_csv(
-        f"./llm-perf-dataset/reports/additional_data.csv")
+    bench_df = pd.read_csv(f"./llm-perf-dataset/reports/{benchmark}.csv")
+    scores_df = pd.read_csv(f"./llm-perf-dataset/reports/additional_data.csv")
     bench_df = bench_df.merge(scores_df, on="model", how="left")
 
     return bench_df
 
 
 def get_benchmark_table(bench_df):
-
     # filter
     bench_df = bench_df[list(COLUMNS_MAPPING.keys())]
     # rename
@@ -54,73 +78,76 @@ def get_benchmark_table(bench_df):
     bench_df.sort_values(by=SORTING_COLUMN, ascending=False, inplace=True)
     # transform
     bench_df["Model 🤗"] = bench_df["Model 🤗"].apply(make_clickable_model)
+    bench_df["#️⃣ Parameters 📏"] = bench_df["#️⃣ Parameters 📏"].apply(num_to_str)
     bench_df["Average Open LLM Score ⬆️"] = bench_df["Average Open LLM Score ⬆️"].apply(
-        make_clickable_score)
-    bench_df["#Parameters 📏"] = bench_df["#Parameters 📏"].apply(num_to_str)
-
+        make_clickable_score
+    )
     return bench_df
 
 
 def get_benchmark_plot(bench_df):
-
     # untill falcon gets fixed / natively supported
     bench_df = bench_df[bench_df["generate.latency(s)"] < 100]
 
     fig = px.scatter(
-        bench_df, x="generate.latency(s)", y="average",
-        color='model_type', symbol='backend.name', size='forward.peak_memory(MB)',
-        custom_data=['model', 'backend.name', 'backend.torch_dtype',
-                     'forward.peak_memory(MB)', 'generate.throughput(tokens/s)'],
-        symbol_sequence=['triangle-up', 'circle'],
+        bench_df,
+        x="generate.latency(s)",
+        y="average",
+        color="model_type",
+        symbol="backend.name",
+        size="forward.peak_memory(MB)",
+        custom_data=[
+            "model",
+            "backend.name",
+            "backend.torch_dtype",
+            "forward.peak_memory(MB)",
+            "generate.throughput(tokens/s)",
+        ],
+        symbol_sequence=["triangle-up", "circle"],
         # as many distinct colors as there are model_type,backend.name couples
         color_discrete_sequence=px.colors.qualitative.Light24,
     )
 
     fig.update_layout(
         title={
-            'text': "Model Score vs. Latency vs. Memory",
-            'y': 0.95, 'x': 0.5,
-            'xanchor': 'center',
-            'yanchor': 'top'
+            "text": "Model Score vs. Latency vs. Memory",
+            "y": 0.95,
+            "x": 0.5,
+            "xanchor": "center",
+            "yanchor": "top",
         },
         xaxis_title="Per 1000 Tokens Latency (s)",
         yaxis_title="Average Open LLM Score",
         legend_title="Model Type and Backend",
         width=1200,
         height=600,
-        # legend=dict(
-        #     orientation="h",
-        #     yanchor="bottom",
-        #     y=-0.35,
-        #     xanchor="center",
-        #     x=0.5
-        # )
     )
 
     fig.update_traces(
-        hovertemplate="<br>".join([
-            "Model: %{customdata[0]}",
-            "Backend: %{customdata[1]}",
-            "Datatype: %{customdata[2]}",
-            "Peak Memory (MB): %{customdata[3]}",
-            "Throughput (tokens/s): %{customdata[4]}",
-            "Average Open LLM Score: %{y}",
-            "Per 1000 Tokens Latency (s): %{x}",
-        ])
+        hovertemplate="<br>".join(
+            [
+                "Model: %{customdata[0]}",
+                "Backend: %{customdata[1]}",
+                "Datatype: %{customdata[2]}",
+                "Peak Memory (MB): %{customdata[3]}",
+                "Throughput (tokens/s): %{customdata[4]}",
+                "Average Open LLM Score: %{y}",
+                "Per 1000 Tokens Latency (s): %{x}",
+            ]
+        )
     )
 
     return fig
 
 
 def filter_query(text, backends, datatypes, threshold, benchmark="1xA100-80GB"):
-
     raw_df = get_benchmark_df(benchmark=benchmark)
 
     filtered_df = raw_df[
-        raw_df["model"].str.lower().str.contains(text.lower()) &
-        raw_df["backend.name"].isin(backends) &
-        raw_df["backend.torch_dtype"].isin(datatypes) &
-        (raw_df["average"] >= threshold)
+        raw_df["model"].str.lower().str.contains(text.lower())
+        & raw_df["backend.name"].isin(backends)
+        & raw_df["backend.torch_dtype"].isin(datatypes)
+        & (raw_df["average"] >= threshold)
     ]
 
     filtered_table = get_benchmark_table(filtered_df)
@@ -221,8 +248,12 @@ with demo:
 
 # Restart space every hour
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=3600,
-                  args=[LLM_PERF_LEADERBOARD_REPO, OPTIMUM_TOKEN])
+scheduler.add_job(
+    restart_space,
+    "interval",
+    seconds=3600,
+    args=[LLM_PERF_LEADERBOARD_REPO, OPTIMUM_TOKEN],
+)
 scheduler.start()
 
 # Launch demo