Update curated.py

curated.py

@@ -694,6 +694,53 @@ def get_chart_28168342():
     return fig
 
 
+def get_chart_new():
+    fig = go.Figure()
+    filter_names = [
+        "Download",
+        "Language",
+        "Min word count",
+        "Title Abstract",
+        "Majority language",
+        "Paragraph count",
+        "Frequency",
+        "Unigram log probability",
+        "Local dedup",
+    ]
+
+    data_sources = [
+        ("Wikipedia", [61614907, 0, 1146416, 0, 0, 0, 0, 0, 20]),
+        ("Freelaw", [75971288, 2280522, 5518932, 0, 0, 0, 0, 48660, 20]),
+        ("DM Maths", [112559888, 0, 0, 0, 0, 0, 0, 0, 20]),
+        ("USPTO", [6880276, 1312, 129042, 0, 0, 0, 0, 533, 20]),
+        ("PG19", [28752, 69, 1, 0, 0, 0, 0, 50, 20]),
+        ("Hackernews", [2064931, 54129, 314, 0, 0, 0, 0, 6852, 20]),
+        ("Ubuntu IRC", [37966, 14465, 33, 0, 0, 0, 0, 263, 20]),
+        ("Europarl", [69814, 0, 0, 0, 0, 0, 0, 0, 20]),
+        ("StackExchange", [23246548, 0, 196, 0, 0, 0, 0, 0, 20]),
+        ("Arxiv", [1911867, 42426, 105601, 0, 0, 0, 0, 1179, 20]),
+        ("S2ORC", [12963563, 0, 0, 2232450, 1275493, 148804, 1251669, 0, 20]),
+        ("S2ORC Abstract", [102324176, 18456575, 978308, 0, 0, 0, 0, 111381, 20]),
+        ("PubMed Central", [5230932, 400446, 62176, 0, 0, 0, 0, 836, 20]),
+        ("PubMed Central Abstract", [25787474, 3100, 36419, 0, 0, 0, 0, 1231, 20]),
+        ("PhilPapers", [49389, 10214, 0, 0, 0, 0, 0, 47, 20]),
+    ]
+
+    for name, x_values in data_sources:
+        fig.add_trace(
+            go.Funnel(
+                name=name,
+                orientation="h",
+                y=filter_names,
+                x=x_values,
+                textinfo="value+percent total",
+                textposition="inside",
+            )
+        )
+
+    fig.update_layout(height=500, plot_bgcolor="rgba(0,0,0,0)")
+    return fig
+
 def update(target: str, request):
     params = request.query_params
     if data_source := params.get(f"data_source_{target}"):
@@ -836,6 +883,7 @@ def curated(request):
             table_desc,
             data_preprocessing_div,
             plotly2fasthtml(get_chart_28168342()),
+            plotly2fasthtml(get_chart_new()),
             H2("Curated Sources Processing"),
             filtering_process,
             data_preparation_div,