updated ryan science tags

all_rich_dataset_cards.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:94ac600eb5100aa7acaeeec3d05becbee7ac11eba9595a0f9e38286879285349
-size 5475858

app.py

@@ -1,17 +1,122 @@
+# import gradio as gr
+# import polars as pl
+
+# # Paths or HF Hub URLs for Parquet files
+# RICH_PARQUET_PATH = "all_rich_dataset_cards.parquet"
+# MISSING_PARQUET_PATH = "all_minimal_dataset_cards.parquet"
+
+# ROWS_PER_PAGE = 50
+
+# # Lazy load datasets
+# lazy_rich = pl.scan_parquet(RICH_PARQUET_PATH)
+# lazy_missing = pl.scan_parquet(MISSING_PARQUET_PATH)
+
+# current_lazy_df = lazy_missing  # Default dataset
+
+# # Helper function to fetch a page
+# def get_page(lazy_df: pl.LazyFrame, page: int, column: str = None, query: str = ""):
+#     filtered_df = lazy_df
+#     if column and query:
+#         query_lower = query.lower().strip()
+#         # Case-insensitive search
+#         filtered_df = filtered_df.with_columns([
+#             pl.col(column).cast(pl.Utf8).str.to_lowercase().alias(column)
+#         ]).filter(pl.col(column).str.contains(query_lower, literal=False))
+#     start = page * ROWS_PER_PAGE
+#     page_df = filtered_df.slice(start, ROWS_PER_PAGE).collect().to_pandas()
+#     total_rows = filtered_df.collect().height
+#     total_pages = (total_rows - 1) // ROWS_PER_PAGE + 1
+#     return page_df, total_pages
+
+# # Initialize first page
+# initial_df, total_pages = get_page(current_lazy_df, 0)
+# columns = list(initial_df.columns)
+
+# with gr.Blocks() as demo:
+#     gr.Markdown("## Dataset Insight Portal")
+
+#     # Dataset selection
+#     dataset_select = gr.Dropdown(
+#         choices=["DatasetCards rich in information", "DatasetCards missing information"],
+#         value="DatasetCards missing information",
+#         label="Select Dataset"
+#     )
+
+#     # Pagination controls
+#     with gr.Row():
+#         prev_btn = gr.Button("Previous", elem_id="small-btn")
+#         next_btn = gr.Button("Next", elem_id="small-btn")
+#         page_number = gr.Number(value=0, label="Page", precision=0)
+#         total_pages_display = gr.Label(value=f"Total Pages: {total_pages}")
+
+#     # Data table
+#     data_table = gr.Dataframe(
+#         value=initial_df, headers=columns, datatype="str",
+#         interactive=False, row_count=ROWS_PER_PAGE
+#     )
+
+#     # Column search
+#     with gr.Row():
+#         col_dropdown = gr.Dropdown(choices=columns, label="Column")
+#         search_text = gr.Textbox(label="Search")
+#         search_btn = gr.Button("Search", elem_id="small-btn")
+#         reset_btn = gr.Button("Reset", elem_id="small-btn")
+
+#     # --- Functions ---
+#     def load_dataset(dataset_choice):
+#         global current_lazy_df
+#         current_lazy_df = lazy_rich if dataset_choice == "DatasetCards rich in information" else lazy_missing
+#         initial_df, total_pages = get_page(current_lazy_df, 0)
+#         columns = list(initial_df.columns)
+#         return (
+#             gr.update(value=initial_df, headers=columns),
+#             f"Total Pages: {total_pages}",
+#             0,
+#             gr.update(choices=columns, value=columns[0])
+#         )
+
+#     def next_page_func(page, column, query):
+#         page += 1
+#         page_df, total_pages = get_page(current_lazy_df, page, column, query)
+#         if page >= total_pages:
+#             page = total_pages - 1
+#             page_df, total_pages = get_page(current_lazy_df, page, column, query)
+#         return page_df, f"Total Pages: {total_pages}", page
+
+#     def prev_page_func(page, column, query):
+#         page -= 1
+#         page = max(0, page)
+#         page_df, total_pages = get_page(current_lazy_df, page, column, query)
+#         return page_df, f"Total Pages: {total_pages}", page
+
+#     def search_func(column, query):
+#         page_df, total_pages = get_page(current_lazy_df, 0, column, query)
+#         return page_df, f"Total Pages: {total_pages}", 0
+
+#     def reset_func():
+#         page_df, total_pages = get_page(current_lazy_df, 0)
+#         return page_df, f"Total Pages: {total_pages}", 0
+
+#     # --- Event Listeners ---
+#     dataset_select.change(load_dataset, dataset_select, [data_table, total_pages_display, page_number, col_dropdown])
+#     next_btn.click(next_page_func, [page_number, col_dropdown, search_text], [data_table, total_pages_display, page_number])
+#     prev_btn.click(prev_page_func, [page_number, col_dropdown, search_text], [data_table, total_pages_display, page_number])
+#     search_btn.click(search_func, [col_dropdown, search_text], [data_table, total_pages_display, page_number])
+#     reset_btn.click(reset_func, [], [data_table, total_pages_display, page_number])
+
+# demo.launch()
+
+
 import gradio as gr
 import polars as pl
 
-# Paths or HF Hub URLs for Parquet files
-RICH_PARQUET_PATH = "all_rich_dataset_cards.parquet"
-MISSING_PARQUET_PATH = "all_minimal_dataset_cards.parquet"
+# Path for the combined Parquet file
+COMBINED_PARQUET_PATH = "datasetcards.parquet"
 
 ROWS_PER_PAGE = 50
 
-# Lazy load datasets
-lazy_rich = pl.scan_parquet(RICH_PARQUET_PATH)
-lazy_missing = pl.scan_parquet(MISSING_PARQUET_PATH)
-
-current_lazy_df = lazy_missing  # Default dataset
+# Lazy load dataset
+lazy_df = pl.scan_parquet(COMBINED_PARQUET_PATH)
 
 # Helper function to fetch a page
 def get_page(lazy_df: pl.LazyFrame, page: int, column: str = None, query: str = ""):
@@ -29,18 +134,13 @@ def get_page(lazy_df: pl.LazyFrame, page: int, column: str = None, query: str =
     return page_df, total_pages
 
 # Initialize first page
-initial_df, total_pages = get_page(current_lazy_df, 0)
+initial_df, total_pages = get_page(lazy_df, 0)
 columns = list(initial_df.columns)
 
 with gr.Blocks() as demo:
     gr.Markdown("## Dataset Insight Portal")
-
-    # Dataset selection
-    dataset_select = gr.Dropdown(
-        choices=["DatasetCards rich in information", "DatasetCards missing information"],
-        value="DatasetCards missing information",
-        label="Select Dataset"
-    )
+    gr.Markdown("This space allows you to explore the combined dataset of DatasetCards. "
+                "You can navigate pages, search within columns, and inspect the dataset easily.")
 
     # Pagination controls
     with gr.Row():
@@ -63,17 +163,7 @@ with gr.Blocks() as demo:
         reset_btn = gr.Button("Reset", elem_id="small-btn")
 
     # --- Functions ---
-    def load_dataset(dataset_choice):
-        global current_lazy_df
-        current_lazy_df = lazy_rich if dataset_choice == "DatasetCards rich in information" else lazy_missing
-        initial_df, total_pages = get_page(current_lazy_df, 0)
-        columns = list(initial_df.columns)
-        return (
-            gr.update(value=initial_df, headers=columns),
-            f"Total Pages: {total_pages}",
-            0,
-            gr.update(choices=columns, value=columns[0])
-        )
+    current_lazy_df = lazy_df  # single dataset
 
     def next_page_func(page, column, query):
         page += 1
@@ -98,7 +188,6 @@ with gr.Blocks() as demo:
         return page_df, f"Total Pages: {total_pages}", 0
 
     # --- Event Listeners ---
-    dataset_select.change(load_dataset, dataset_select, [data_table, total_pages_display, page_number, col_dropdown])
     next_btn.click(next_page_func, [page_number, col_dropdown, search_text], [data_table, total_pages_display, page_number])
     prev_btn.click(prev_page_func, [page_number, col_dropdown, search_text], [data_table, total_pages_display, page_number])
     search_btn.click(search_func, [col_dropdown, search_text], [data_table, total_pages_display, page_number])

all_minimal_dataset_cards.parquet → datasetcards.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5adb59f94fb6f08f5c0859e21e55ed56ec40f40d9cde349427bf24065e775d60
-size 17318878
+oid sha256:c248074b63bc77b236e8096e3423779f3a5bf4cbe24a2683ea63da31a1c4c154
+size 35038132

heuristic_approach.py

@@ -0,0 +1,277 @@
+# from huggingface_hub import list_datasets, DatasetCard
+# import re
+# import pandas as pd
+# import os
+# import time
+# import random
+# from concurrent.futures import ThreadPoolExecutor, as_completed
+# from requests.exceptions import HTTPError
+
+
+# # ---------- Retry helper ----------
+# def retry_load_card(dataset_id, retries=5, base_wait=60):
+#     """
+#     Try to load a dataset card with retries if 429 (rate limit) occurs.
+#     Uses Retry-After header if available, otherwise exponential backoff.
+#     """
+#     for attempt in range(retries):
+#         try:
+#             return DatasetCard.load(dataset_id)
+#         except HTTPError as e:
+#             if e.response is not None and e.response.status_code == 429:
+#                 wait_time = e.response.headers.get("Retry-After")
+#                 if wait_time is not None:
+#                     wait_time = int(wait_time)
+#                 else:
+#                     wait_time = base_wait * (2 ** attempt) + random.randint(0, 10)
+#                 print(f"[429] Rate limit hit for {dataset_id}. Sleeping {wait_time}s (attempt {attempt+1}/{retries})...")
+#                 time.sleep(wait_time)
+#                 continue
+#             else:
+#                 raise  # don't retry for other HTTP errors
+#         except Exception as e:
+#             print(f"[ERROR] {dataset_id}: {e}")
+#             raise
+#     raise RuntimeError(f"Failed to load {dataset_id} after {retries} retries.")
+
+
+# # ---------- Heuristic functions with reasons ----------
+# def check_card_quality(card_text, metadata, dataset_url):
+#     reasons = []
+#     length = len(card_text)
+#     word_count = len(card_text.split())
+
+#     if metadata is None or len(metadata) == 0:
+#         print(length, word_count, dataset_url)
+#         if length < 200:
+#             reasons.append("No metadata and no description")
+#             return "minimal", reasons, word_count
+#         else:
+#             reasons.append("No metadata but has description")
+#             return "minimal", reasons, word_count
+#     else:
+#         if length < 200:
+#             reasons.append(f"Short description (char count={length}, words={word_count})")
+#             return "minimal", reasons, word_count
+#         else:
+#             return "rich", reasons, word_count
+    
+# # ---------- Worker function for one dataset ----------
+# def process_dataset(ds, save_dir):
+#     try:
+#         card = retry_load_card(ds.id)
+#         card_text = card.text or ""
+#         metadata = card.data.to_dict() if card.data else {}
+#         dataset_url = f"https://huggingface.co/datasets/{ds.id}"
+
+#         # Save README locally
+#         readme_path = os.path.join(save_dir, f"{ds.id.replace('/', '__')}_README.md")
+#         with open(readme_path, "w", encoding="utf-8") as f:
+#             f.write(card_text)
+
+#         category, reasons, word_count = check_card_quality(card_text, metadata, dataset_url)
+#         row = {
+#             "dataset_id": ds.id,
+#             "dataset_url": dataset_url,
+#             "downloads": getattr(ds, "downloads", None),
+#             "reason": "; ".join(reasons),
+#             "readme_path": readme_path,
+#             "word_count": word_count,
+#             "category": category,
+#         }
+#         return row
+#     except Exception as e:
+#         return {
+#             "dataset_id": ds.id,
+#             "dataset_url": f"https://huggingface.co/datasets/{ds.id}",
+#             "downloads": getattr(ds, "downloads", None),
+#             "reason": f"Failed to load card",
+#             "readme_path": None,
+#             "word_count": 0,
+#             "category": "minimal",
+#         }
+
+
+# # ---------- Main ----------
+# def collect_dataset_ids(limit=1000, save_dir="dataset_readmes", max_workers=16):
+#     minimal_results = []
+#     rich_results = []
+
+#     os.makedirs(save_dir, exist_ok=True)
+
+#     print(f"Fetching up to {limit} datasets (sorted by downloads)...")
+#     datasets = list_datasets()
+
+#     with ThreadPoolExecutor(max_workers=max_workers) as executor:
+#         futures = [executor.submit(process_dataset, ds, save_dir) for ds in datasets]
+#         for i, f in enumerate(as_completed(futures), 1):
+#             row = f.result()
+#             if row["category"] == "minimal":
+#                 minimal_results.append(row)
+#             else:
+#                 rich_results.append(row)
+#     return minimal_results, rich_results
+
+
+# if __name__ == "__main__":
+#     minimal, rich = collect_dataset_ids(limit=1000, max_workers=16)
+
+#     # Save separate CSV files
+#     if minimal:
+#         pd.DataFrame(minimal).to_csv("all_minimal_dataset_cards.csv", index=False)
+#     if rich:
+#         pd.DataFrame(rich).to_csv("all_rich_dataset_cards.csv", index=False)
+
+#     print("\nSaved results to:")
+#     if minimal:
+#         print(" - minimal_dataset_cards.csv")
+#     if rich:
+#         print(" - rich_dataset_cards.csv")
+#     print(" - README files in ./dataset_readmes/")
+
+#     print("\nSummary:")
+#     print(f"Minimal: {len(minimal)}")
+#     print(f"Rich: {len(rich)}")
+
+
+from huggingface_hub import list_datasets, DatasetCard
+import re
+import pandas as pd
+import os
+import time
+import random
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from requests.exceptions import HTTPError
+
+
+# # ---------- Retry helper ----------
+# def retry_load_card(dataset_id, retries=5, base_wait=60):
+#     for attempt in range(retries):
+#         try:
+#             return DatasetCard.load(dataset_id)
+#         except HTTPError as e:
+#             if e.response is not None and e.response.status_code == 429:
+#                 wait_time = e.response.headers.get("Retry-After")
+#                 if wait_time is not None:
+#                     wait_time = int(wait_time)
+#                 else:
+#                     wait_time = base_wait * (2 ** attempt) + random.randint(0, 10)
+#                 print(f"[429] Rate limit hit for {dataset_id}. Sleeping {wait_time}s (attempt {attempt+1}/{retries})...")
+#                 time.sleep(wait_time)
+#                 continue
+#             else:
+#                 raise
+#         except Exception as e:
+#             print(f"[ERROR] {dataset_id}: {e}")
+#             raise
+#     raise RuntimeError(f"Failed to load {dataset_id} after {retries} retries.")
+
+
+# ---------- Heuristic functions with reasons ----------
+def check_card_quality(card_text, metadata, dataset_url):
+    reasons = []
+    length = len(card_text)
+    word_count = len(card_text.split())
+
+    if metadata is None or len(metadata) == 0:
+        print(length, word_count, dataset_url)
+        if length < 200:
+            reasons.append("No metadata and no description")
+            return "minimal", reasons, word_count
+        else:
+            reasons.append("No metadata but has description")
+            return "minimal", reasons, word_count
+    else:
+        if length < 200:
+            reasons.append(f"Short description (char count={length}, words={word_count})")
+            return "minimal", reasons, word_count
+        else:
+            return "rich", reasons, word_count
+    
+
+# ---------- Worker function for one dataset ----------
+def process_dataset(ds, save_dir):
+    try:
+        card = DatasetCard.load(ds.id)
+        card_text = card.text or ""
+        metadata = card.data.to_dict() if card.data else {}
+        dataset_url = f"https://huggingface.co/datasets/{ds.id}"
+
+        # Save README locally
+        readme_path = os.path.join(save_dir, f"{ds.id.replace('/', '__')}_README.md")
+        with open(readme_path, "w", encoding="utf-8") as f:
+            f.write(card_text)
+
+        category, reasons, word_count = check_card_quality(card_text, metadata, dataset_url)
+
+        row = {
+            "dataset_id": ds.id,
+            "dataset_url": dataset_url,
+            "downloads": getattr(ds, "downloads", None),
+            "author": metadata.get("author", None),
+            "license": metadata.get("license", None),
+            "tags": ", ".join(metadata.get("tags", [])) if metadata.get("tags") else None,
+            "task_categories": ", ".join(metadata.get("task_categories", [])) if metadata.get("task_categories") else None,
+            "last_modified": getattr(ds, "lastModified", None),
+            "reason": "; ".join(reasons),
+            "readme_path": readme_path,
+            "word_count": word_count,
+            "category": category,
+        }
+        return row
+    except Exception as e:
+        return {
+            "dataset_id": ds.id,
+            "dataset_url": f"https://huggingface.co/datasets/{ds.id}",
+            "downloads": getattr(ds, "downloads", None),
+            "author": None,
+            "license": None,
+            "tags": None,
+            "task_categories": None,
+            "last_modified": None,
+            "reason": "Failed to load card",
+            "readme_path": None,
+            "word_count": 0,
+            "category": "minimal",
+        }
+
+
+# ---------- Main ----------
+def collect_dataset_ids(save_dir="dataset_readmes", max_workers=16):
+    minimal_results = []
+    rich_results = []
+
+    os.makedirs(save_dir, exist_ok=True)
+
+    datasets = list_datasets()
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_dataset, ds, save_dir) for ds in datasets]
+        for i, f in enumerate(as_completed(futures), 1):
+            row = f.result()
+            if row["category"] == "minimal":
+                minimal_results.append(row)
+            else:
+                rich_results.append(row)
+    return minimal_results, rich_results
+
+
+if __name__ == "__main__":
+    minimal, rich = collect_dataset_ids(limit=100, max_workers=16)
+
+    # Save separate CSV files
+    if minimal:
+        pd.DataFrame(minimal).to_csv("all_minimal_dataset_cards.csv", index=False)
+    if rich:
+        pd.DataFrame(rich).to_csv("all_rich_dataset_cards.csv", index=False)
+
+    print("\nSaved results to:")
+    if minimal:
+        print(" - minimal_dataset_cards.csv")
+    if rich:
+        print(" - rich_dataset_cards.csv")
+    print(" - README files in ./dataset_readmes/")
+
+    print("\nSummary:")
+    print(f"Minimal: {len(minimal)}")
+    print(f"Rich: {len(rich)}")

preprocessing.ipynb

@@ -0,0 +1,2098 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4e64d318",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "                                        dataset_id  \\\n",
+      "0  akjadhav/leandojo-lean4-formal-informal-strings   \n",
+      "1                                     aemska/stuhl   \n",
+      "2                            Pogpotatofarmer/memes   \n",
+      "3     Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+      "4                  chamisfum/brain_tumor_3_classes   \n",
+      "\n",
+      "                                         dataset_url  downloads author  \\\n",
+      "0  https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+      "1       https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+      "2  https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+      "3  https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+      "4  https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+      "\n",
+      "    license  tags task_categories              last_modified  \\\n",
+      "0      None  None            None  2024-01-30 07:40:02+00:00   \n",
+      "1  openrail  None            None  2022-11-11 14:12:36+00:00   \n",
+      "2        cc  None            None  2022-07-15 21:11:34+00:00   \n",
+      "3      None  None            None                       None   \n",
+      "4      None  None            None                       None   \n",
+      "\n",
+      "                                      reason  \\\n",
+      "0             No metadata and no description   \n",
+      "1  Short description (char count=0, words=0)   \n",
+      "2  Short description (char count=0, words=0)   \n",
+      "3                        Failed to load card   \n",
+      "4                        Failed to load card   \n",
+      "\n",
+      "                                         readme_path  word_count category  \n",
+      "0  dataset_readmes/akjadhav__leandojo-lean4-forma...           0  minimal  \n",
+      "1            dataset_readmes/aemska__stuhl_README.md           0  minimal  \n",
+      "2   dataset_readmes/Pogpotatofarmer__memes_README.md           0  minimal  \n",
+      "3                                               None           0  minimal  \n",
+      "4                                               None           0  minimal  \n",
+      "                                          dataset_id  \\\n",
+      "0  autoevaluate/autoeval-staging-eval-launch__gov...   \n",
+      "1  autoevaluate/autoeval-eval-emotion-default-fe1...   \n",
+      "2                                        LTCB/enwik8   \n",
+      "3                           boltuix/emotions-dataset   \n",
+      "4                               yixuantt/MultiHopRAG   \n",
+      "\n",
+      "                                         dataset_url  downloads author  \\\n",
+      "0  https://huggingface.co/datasets/autoevaluate/a...          8   None   \n",
+      "1  https://huggingface.co/datasets/autoevaluate/a...          8   None   \n",
+      "2        https://huggingface.co/datasets/LTCB/enwik8        154   None   \n",
+      "3  https://huggingface.co/datasets/boltuix/emotio...        754   None   \n",
+      "4  https://huggingface.co/datasets/yixuantt/Multi...       7050   None   \n",
+      "\n",
+      "   license                                               tags  \\\n",
+      "0     None                              autotrain, evaluation   \n",
+      "1     None                              autotrain, evaluation   \n",
+      "2  ['mit']                                               None   \n",
+      "3      mit  emotions, nlp, sentiment-analysis, emotion-cla...   \n",
+      "4   odc-by                                               None   \n",
+      "\n",
+      "                          task_categories              last_modified reason  \\\n",
+      "0                                    None  2022-09-09 07:44:04+00:00   None   \n",
+      "1                                    None  2022-09-16 20:22:59+00:00   None   \n",
+      "2              fill-mask, text-generation  2024-01-18 11:19:13+00:00   None   \n",
+      "3                                    None  2025-05-25 15:41:59+00:00   None   \n",
+      "4  question-answering, feature-extraction  2024-01-30 02:49:29+00:00   None   \n",
+      "\n",
+      "                                         readme_path  word_count category  \n",
+      "0  dataset_readmes/autoevaluate__autoeval-staging...          55     rich  \n",
+      "1  dataset_readmes/autoevaluate__autoeval-eval-em...          57     rich  \n",
+      "2             dataset_readmes/LTCB__enwik8_README.md         427     rich  \n",
+      "3  dataset_readmes/boltuix__emotions-dataset_READ...        1643     rich  \n",
+      "4    dataset_readmes/yixuantt__MultiHopRAG_README.md         111     rich  \n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Read parquet files\n",
+    "df1 = pd.read_parquet(\"/home/santosh/Repositories/personal/huggingface/dataset-insight-portal/all_minimal_dataset_cards.parquet\")\n",
+    "df2 = pd.read_parquet(\"/home/santosh/Repositories/personal/huggingface/dataset-insight-portal/all_rich_dataset_cards.parquet\")\n",
+    "\n",
+    "# Display first few rows\n",
+    "print(df1.head())\n",
+    "print(df2.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "e9a20931",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset_id</th>\n",
+       "      <th>dataset_url</th>\n",
+       "      <th>downloads</th>\n",
+       "      <th>author</th>\n",
+       "      <th>license</th>\n",
+       "      <th>tags</th>\n",
+       "      <th>task_categories</th>\n",
+       "      <th>last_modified</th>\n",
+       "      <th>reason</th>\n",
+       "      <th>readme_path</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>category</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "      <td>https://huggingface.co/datasets/akjadhav/leand...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2024-01-30 07:40:02+00:00</td>\n",
+       "      <td>No metadata and no description</td>\n",
+       "      <td>dataset_readmes/akjadhav__leandojo-lean4-forma...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "      <td>https://huggingface.co/datasets/aemska/stuhl</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>openrail</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-11-11 14:12:36+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/aemska__stuhl_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Pogpotatofarmer/memes</td>\n",
+       "      <td>https://huggingface.co/datasets/Pogpotatofarme...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cc</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-07-15 21:11:34+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Pogpotatofarmer__memes_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "      <td>https://huggingface.co/datasets/Splend1dchan/N...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "      <td>https://huggingface.co/datasets/chamisfum/brai...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>400292</th>\n",
+       "      <td>TAUR-dev/D-EVAL__standard_eval_v3__RC_BF_ab-bo...</td>\n",
+       "      <td>https://huggingface.co/datasets/TAUR-dev/D-EVA...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2025-09-19 06:27:52+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/TAUR-dev__D-EVAL__standard_eva...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>400293</th>\n",
+       "      <td>TAUR-dev/D-EVAL__standard_eval_v3__RC_BF_ab-bo...</td>\n",
+       "      <td>https://huggingface.co/datasets/TAUR-dev/D-EVA...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2025-09-19 06:28:16+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/TAUR-dev__D-EVAL__standard_eva...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>400294</th>\n",
+       "      <td>haru101/Minecraft-Knowledge-Dataset</td>\n",
+       "      <td>https://huggingface.co/datasets/haru101/Minecr...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>question-answering</td>\n",
+       "      <td>2025-09-19 06:33:33+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/haru101__Minecraft-Knowledge-D...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>400295</th>\n",
+       "      <td>sxj1215/mmimdb_sorted_with_label_2</td>\n",
+       "      <td>https://huggingface.co/datasets/sxj1215/mmimdb...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2025-09-19 06:35:25+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/sxj1215__mmimdb_sorted_with_la...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>400296</th>\n",
+       "      <td>Vikir2411CS19/Multimodal_Complaint</td>\n",
+       "      <td>https://huggingface.co/datasets/Vikir2411CS19/...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2025-09-19 06:35:01+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Vikir2411CS19__Multimodal_Comp...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>400297 rows × 12 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                               dataset_id  \\\n",
+       "0         akjadhav/leandojo-lean4-formal-informal-strings   \n",
+       "1                                            aemska/stuhl   \n",
+       "2                                   Pogpotatofarmer/memes   \n",
+       "3            Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+       "4                         chamisfum/brain_tumor_3_classes   \n",
+       "...                                                   ...   \n",
+       "400292  TAUR-dev/D-EVAL__standard_eval_v3__RC_BF_ab-bo...   \n",
+       "400293  TAUR-dev/D-EVAL__standard_eval_v3__RC_BF_ab-bo...   \n",
+       "400294                haru101/Minecraft-Knowledge-Dataset   \n",
+       "400295                 sxj1215/mmimdb_sorted_with_label_2   \n",
+       "400296                 Vikir2411CS19/Multimodal_Complaint   \n",
+       "\n",
+       "                                              dataset_url  downloads author  \\\n",
+       "0       https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+       "1            https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+       "2       https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+       "3       https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+       "4       https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+       "...                                                   ...        ...    ...   \n",
+       "400292  https://huggingface.co/datasets/TAUR-dev/D-EVA...          0   None   \n",
+       "400293  https://huggingface.co/datasets/TAUR-dev/D-EVA...          0   None   \n",
+       "400294  https://huggingface.co/datasets/haru101/Minecr...          0   None   \n",
+       "400295  https://huggingface.co/datasets/sxj1215/mmimdb...          0   None   \n",
+       "400296  https://huggingface.co/datasets/Vikir2411CS19/...          0   None   \n",
+       "\n",
+       "           license  tags     task_categories              last_modified  \\\n",
+       "0             None  None                None  2024-01-30 07:40:02+00:00   \n",
+       "1         openrail  None                None  2022-11-11 14:12:36+00:00   \n",
+       "2               cc  None                None  2022-07-15 21:11:34+00:00   \n",
+       "3             None  None                None                       None   \n",
+       "4             None  None                None                       None   \n",
+       "...            ...   ...                 ...                        ...   \n",
+       "400292        None  None                None  2025-09-19 06:27:52+00:00   \n",
+       "400293        None  None                None  2025-09-19 06:28:16+00:00   \n",
+       "400294  apache-2.0  None  question-answering  2025-09-19 06:33:33+00:00   \n",
+       "400295        None  None                None  2025-09-19 06:35:25+00:00   \n",
+       "400296        None  None                None  2025-09-19 06:35:01+00:00   \n",
+       "\n",
+       "                                           reason  \\\n",
+       "0                  No metadata and no description   \n",
+       "1       Short description (char count=0, words=0)   \n",
+       "2       Short description (char count=0, words=0)   \n",
+       "3                             Failed to load card   \n",
+       "4                             Failed to load card   \n",
+       "...                                           ...   \n",
+       "400292  Short description (char count=0, words=0)   \n",
+       "400293  Short description (char count=0, words=0)   \n",
+       "400294  Short description (char count=0, words=0)   \n",
+       "400295  Short description (char count=0, words=0)   \n",
+       "400296  Short description (char count=0, words=0)   \n",
+       "\n",
+       "                                              readme_path  word_count category  \n",
+       "0       dataset_readmes/akjadhav__leandojo-lean4-forma...           0  minimal  \n",
+       "1                 dataset_readmes/aemska__stuhl_README.md           0  minimal  \n",
+       "2        dataset_readmes/Pogpotatofarmer__memes_README.md           0  minimal  \n",
+       "3                                                    None           0  minimal  \n",
+       "4                                                    None           0  minimal  \n",
+       "...                                                   ...         ...      ...  \n",
+       "400292  dataset_readmes/TAUR-dev__D-EVAL__standard_eva...           0  minimal  \n",
+       "400293  dataset_readmes/TAUR-dev__D-EVAL__standard_eva...           0  minimal  \n",
+       "400294  dataset_readmes/haru101__Minecraft-Knowledge-D...           0  minimal  \n",
+       "400295  dataset_readmes/sxj1215__mmimdb_sorted_with_la...           0  minimal  \n",
+       "400296  dataset_readmes/Vikir2411CS19__Multimodal_Comp...           0  minimal  \n",
+       "\n",
+       "[400297 rows x 12 columns]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b5582c36",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>url</th>\n",
+       "      <th>field</th>\n",
+       "      <th>keyword</th>\n",
+       "      <th>missing_readme</th>\n",
+       "      <th>missing_card</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>solomonk/reddit_mental_health_posts</td>\n",
+       "      <td>https://huggingface.co/datasets/solomonk/reddi...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>health</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Kira-Asimov/gender_clinical_trial</td>\n",
+       "      <td>https://huggingface.co/datasets/Kira-Asimov/ge...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>clinical</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>samhog/psychology-6k</td>\n",
+       "      <td>https://huggingface.co/datasets/samhog/psychol...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>psychology</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>TCMLM/real_clinical_cases_of_Famous_Old_TCM_Do...</td>\n",
+       "      <td>https://huggingface.co/datasets/TCMLM/real_cli...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>clinical</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>jibrand/plant-dataset-JSONL</td>\n",
+       "      <td>https://huggingface.co/datasets/jibrand/plant-...</td>\n",
+       "      <td>agriculture_and_biology</td>\n",
+       "      <td>plant</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4035</th>\n",
+       "      <td>AshwinManohar/medicine_normalizer_alpaca</td>\n",
+       "      <td>https://huggingface.co/datasets/AshwinManohar/...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>medicine</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4036</th>\n",
+       "      <td>AshwinManohar/medicine_parser_alpaca</td>\n",
+       "      <td>https://huggingface.co/datasets/AshwinManohar/...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>medicine</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4037</th>\n",
+       "      <td>AshwinManohar/medicine_normalizer_alpaca_20k</td>\n",
+       "      <td>https://huggingface.co/datasets/AshwinManohar/...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>medicine</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4038</th>\n",
+       "      <td>Adithyaaaa/plant_leaf_classification</td>\n",
+       "      <td>https://huggingface.co/datasets/Adithyaaaa/pla...</td>\n",
+       "      <td>agriculture_and_biology</td>\n",
+       "      <td>plant</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4039</th>\n",
+       "      <td>benali-ai-24/drug-data-public</td>\n",
+       "      <td>https://huggingface.co/datasets/benali-ai-24/d...</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>drug</td>\n",
+       "      <td>True</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4040 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                     id  \\\n",
+       "0                   solomonk/reddit_mental_health_posts   \n",
+       "1                     Kira-Asimov/gender_clinical_trial   \n",
+       "2                                  samhog/psychology-6k   \n",
+       "3     TCMLM/real_clinical_cases_of_Famous_Old_TCM_Do...   \n",
+       "4                           jibrand/plant-dataset-JSONL   \n",
+       "...                                                 ...   \n",
+       "4035           AshwinManohar/medicine_normalizer_alpaca   \n",
+       "4036               AshwinManohar/medicine_parser_alpaca   \n",
+       "4037       AshwinManohar/medicine_normalizer_alpaca_20k   \n",
+       "4038               Adithyaaaa/plant_leaf_classification   \n",
+       "4039                      benali-ai-24/drug-data-public   \n",
+       "\n",
+       "                                                    url  \\\n",
+       "0     https://huggingface.co/datasets/solomonk/reddi...   \n",
+       "1     https://huggingface.co/datasets/Kira-Asimov/ge...   \n",
+       "2     https://huggingface.co/datasets/samhog/psychol...   \n",
+       "3     https://huggingface.co/datasets/TCMLM/real_cli...   \n",
+       "4     https://huggingface.co/datasets/jibrand/plant-...   \n",
+       "...                                                 ...   \n",
+       "4035  https://huggingface.co/datasets/AshwinManohar/...   \n",
+       "4036  https://huggingface.co/datasets/AshwinManohar/...   \n",
+       "4037  https://huggingface.co/datasets/AshwinManohar/...   \n",
+       "4038  https://huggingface.co/datasets/Adithyaaaa/pla...   \n",
+       "4039  https://huggingface.co/datasets/benali-ai-24/d...   \n",
+       "\n",
+       "                        field     keyword  missing_readme  missing_card  \n",
+       "0               life_sciences      health           False          True  \n",
+       "1               life_sciences    clinical           False          True  \n",
+       "2               life_sciences  psychology            True          True  \n",
+       "3               life_sciences    clinical           False          True  \n",
+       "4     agriculture_and_biology       plant            True          True  \n",
+       "...                       ...         ...             ...           ...  \n",
+       "4035            life_sciences    medicine            True          True  \n",
+       "4036            life_sciences    medicine            True          True  \n",
+       "4037            life_sciences    medicine            True          True  \n",
+       "4038  agriculture_and_biology       plant            True          True  \n",
+       "4039            life_sciences        drug            True          True  \n",
+       "\n",
+       "[4040 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "csv_df = pd.read_csv(\"/home/santosh/Repositories/personal/huggingface/dataset-insight-portal/ds_missing_sci_data_4k.csv\")\n",
+    "csv_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "a061659a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset_id</th>\n",
+       "      <th>dataset_url</th>\n",
+       "      <th>downloads</th>\n",
+       "      <th>author</th>\n",
+       "      <th>license</th>\n",
+       "      <th>tags</th>\n",
+       "      <th>task_categories</th>\n",
+       "      <th>last_modified</th>\n",
+       "      <th>reason</th>\n",
+       "      <th>readme_path</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>category</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "      <td>https://huggingface.co/datasets/akjadhav/leand...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2024-01-30 07:40:02+00:00</td>\n",
+       "      <td>No metadata and no description</td>\n",
+       "      <td>dataset_readmes/akjadhav__leandojo-lean4-forma...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "      <td>https://huggingface.co/datasets/aemska/stuhl</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>openrail</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-11-11 14:12:36+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/aemska__stuhl_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Pogpotatofarmer/memes</td>\n",
+       "      <td>https://huggingface.co/datasets/Pogpotatofarme...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cc</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-07-15 21:11:34+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Pogpotatofarmer__memes_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "      <td>https://huggingface.co/datasets/Splend1dchan/N...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "      <td>https://huggingface.co/datasets/chamisfum/brai...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503185</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:28:15+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503186</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:29:40+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503187</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_5</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:30:53+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503188</th>\n",
+       "      <td>chenxing1234567890/eval_testZ1.2.1</td>\n",
+       "      <td>https://huggingface.co/datasets/chenxing123456...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, tutorial</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:11+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/chenxing1234567890__eval_testZ...</td>\n",
+       "      <td>231</td>\n",
+       "      <td>rich</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503189</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_6</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:09+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>503190 rows × 12 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             dataset_id  \\\n",
+       "0       akjadhav/leandojo-lean4-formal-informal-strings   \n",
+       "1                                          aemska/stuhl   \n",
+       "2                                 Pogpotatofarmer/memes   \n",
+       "3          Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+       "4                       chamisfum/brain_tumor_3_classes   \n",
+       "...                                                 ...   \n",
+       "503185  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14   \n",
+       "503186  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15   \n",
+       "503187              Dongkkka/ffw_bg2_rev4_custom_0919_5   \n",
+       "503188               chenxing1234567890/eval_testZ1.2.1   \n",
+       "503189              Dongkkka/ffw_bg2_rev4_custom_0919_6   \n",
+       "\n",
+       "                                              dataset_url  downloads author  \\\n",
+       "0       https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+       "1            https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+       "2       https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+       "3       https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+       "4       https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+       "...                                                   ...        ...    ...   \n",
+       "503185  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503186  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503187  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "503188  https://huggingface.co/datasets/chenxing123456...          0   None   \n",
+       "503189  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "\n",
+       "           license                                   tags task_categories  \\\n",
+       "0             None                                   None            None   \n",
+       "1         openrail                                   None            None   \n",
+       "2               cc                                   None            None   \n",
+       "3             None                                   None            None   \n",
+       "4             None                                   None            None   \n",
+       "...            ...                                    ...             ...   \n",
+       "503185  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503186  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503187  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503188  apache-2.0                      LeRobot, tutorial        robotics   \n",
+       "503189  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "\n",
+       "                    last_modified                                     reason  \\\n",
+       "0       2024-01-30 07:40:02+00:00             No metadata and no description   \n",
+       "1       2022-11-11 14:12:36+00:00  Short description (char count=0, words=0)   \n",
+       "2       2022-07-15 21:11:34+00:00  Short description (char count=0, words=0)   \n",
+       "3                            None                        Failed to load card   \n",
+       "4                            None                        Failed to load card   \n",
+       "...                           ...                                        ...   \n",
+       "503185  2025-09-19 06:28:15+00:00                                       None   \n",
+       "503186  2025-09-19 06:29:40+00:00                                       None   \n",
+       "503187  2025-09-19 06:30:53+00:00                                       None   \n",
+       "503188  2025-09-19 06:34:11+00:00                                       None   \n",
+       "503189  2025-09-19 06:34:09+00:00                                       None   \n",
+       "\n",
+       "                                              readme_path  word_count category  \n",
+       "0       dataset_readmes/akjadhav__leandojo-lean4-forma...           0  minimal  \n",
+       "1                 dataset_readmes/aemska__stuhl_README.md           0  minimal  \n",
+       "2        dataset_readmes/Pogpotatofarmer__memes_README.md           0  minimal  \n",
+       "3                                                    None           0  minimal  \n",
+       "4                                                    None           0  minimal  \n",
+       "...                                                   ...         ...      ...  \n",
+       "503185  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299     rich  \n",
+       "503186  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299     rich  \n",
+       "503187  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299     rich  \n",
+       "503188  dataset_readmes/chenxing1234567890__eval_testZ...         231     rich  \n",
+       "503189  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299     rich  \n",
+       "\n",
+       "[503190 rows x 12 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df = pd.concat([df1, df2], ignore_index=True)\n",
+    "merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "e0623157",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(11, 7)\n",
+      "                                   id  \\\n",
+      "623              introspector/unimath   \n",
+      "766   ekim15/bone_marrow_cell_dataset   \n",
+      "1645          fabriciojm/ecg-examples   \n",
+      "3280              ahork/record-test-6   \n",
+      "3281       RickRain/SecondTrySimData3   \n",
+      "\n",
+      "                                                    url  \\\n",
+      "623   https://huggingface.co/datasets/introspector/u...   \n",
+      "766   https://huggingface.co/datasets/ekim15/bone_ma...   \n",
+      "1645  https://huggingface.co/datasets/fabriciojm/ecg...   \n",
+      "3280  https://huggingface.co/datasets/ahork/record-t...   \n",
+      "3281  https://huggingface.co/datasets/RickRain/Secon...   \n",
+      "\n",
+      "                           field   keyword  missing_readme  missing_card  \\\n",
+      "623   mathematics_and_statistics      math           False          True   \n",
+      "766                life_sciences   biology            True         False   \n",
+      "1645               life_sciences   medical            True         False   \n",
+      "3280  engineering_and_technology  robotics            True         False   \n",
+      "3281  engineering_and_technology  robotics            True         False   \n",
+      "\n",
+      "                            _id_lower  \n",
+      "623              introspector/unimath  \n",
+      "766   ekim15/bone_marrow_cell_dataset  \n",
+      "1645          fabriciojm/ecg-examples  \n",
+      "3280              ahork/record-test-6  \n",
+      "3281       rickrain/secondtrysimdata3  \n"
+     ]
+    }
+   ],
+   "source": [
+    "# Create lowercase helper columns\n",
+    "df1[\"_dataset_id_lower\"] = df1[\"dataset_id\"].str.lower()\n",
+    "csv_df[\"_id_lower\"] = csv_df[\"id\"].str.lower()\n",
+    "\n",
+    "# Get the rows from df3 where id is NOT in df1\n",
+    "df3_missed = csv_df[~csv_df[\"_id_lower\"].isin(df1[\"_dataset_id_lower\"])]\n",
+    "\n",
+    "print(df3_missed.shape)\n",
+    "print(df3_missed.head())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "b6dbce79",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([['introspector/unimath',\n",
+       "        'https://huggingface.co/datasets/introspector/unimath',\n",
+       "        'mathematics_and_statistics', 'math', False, True,\n",
+       "        'introspector/unimath'],\n",
+       "       ['ekim15/bone_marrow_cell_dataset',\n",
+       "        'https://huggingface.co/datasets/ekim15/bone_marrow_cell_dataset',\n",
+       "        'life_sciences', 'biology', True, False,\n",
+       "        'ekim15/bone_marrow_cell_dataset'],\n",
+       "       ['fabriciojm/ecg-examples',\n",
+       "        'https://huggingface.co/datasets/fabriciojm/ecg-examples',\n",
+       "        'life_sciences', 'medical', True, False,\n",
+       "        'fabriciojm/ecg-examples'],\n",
+       "       ['ahork/record-test-6',\n",
+       "        'https://huggingface.co/datasets/ahork/record-test-6',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'ahork/record-test-6'],\n",
+       "       ['RickRain/SecondTrySimData3',\n",
+       "        'https://huggingface.co/datasets/RickRain/SecondTrySimData3',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'rickrain/secondtrysimdata3'],\n",
+       "       ['MulixBF/record-cube-pick-2cam-black-2',\n",
+       "        'https://huggingface.co/datasets/MulixBF/record-cube-pick-2cam-black-2',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'mulixbf/record-cube-pick-2cam-black-2'],\n",
+       "       ['ricdigi/1two-camera3-test2345',\n",
+       "        'https://huggingface.co/datasets/ricdigi/1two-camera3-test2345',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'ricdigi/1two-camera3-test2345'],\n",
+       "       ['Ninkofu/sushi_put',\n",
+       "        'https://huggingface.co/datasets/Ninkofu/sushi_put',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'ninkofu/sushi_put'],\n",
+       "       ['jokla89/record-test-temp1',\n",
+       "        'https://huggingface.co/datasets/jokla89/record-test-temp1',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'jokla89/record-test-temp1'],\n",
+       "       ['LeRobot-worldwide-hackathon/325-casino-dealer-dice-set',\n",
+       "        'https://huggingface.co/datasets/LeRobot-worldwide-hackathon/325-casino-dealer-dice-set',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'lerobot-worldwide-hackathon/325-casino-dealer-dice-set'],\n",
+       "       ['jackvial/koch_screwdriver_attach_orange_panel_e125',\n",
+       "        'https://huggingface.co/datasets/jackvial/koch_screwdriver_attach_orange_panel_e125',\n",
+       "        'engineering_and_technology', 'robotics', True, False,\n",
+       "        'jackvial/koch_screwdriver_attach_orange_panel_e125']],\n",
+       "      dtype=object)"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df3_missed.values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "0cec2023",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset_id</th>\n",
+       "      <th>dataset_url</th>\n",
+       "      <th>downloads</th>\n",
+       "      <th>author</th>\n",
+       "      <th>license</th>\n",
+       "      <th>tags</th>\n",
+       "      <th>task_categories</th>\n",
+       "      <th>last_modified</th>\n",
+       "      <th>reason</th>\n",
+       "      <th>readme_path</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>category</th>\n",
+       "      <th>_dataset_id_lower</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "      <td>https://huggingface.co/datasets/akjadhav/leand...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2024-01-30 07:40:02+00:00</td>\n",
+       "      <td>No metadata and no description</td>\n",
+       "      <td>dataset_readmes/akjadhav__leandojo-lean4-forma...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "      <td>https://huggingface.co/datasets/aemska/stuhl</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>openrail</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-11-11 14:12:36+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/aemska__stuhl_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Pogpotatofarmer/memes</td>\n",
+       "      <td>https://huggingface.co/datasets/Pogpotatofarme...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cc</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-07-15 21:11:34+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Pogpotatofarmer__memes_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>pogpotatofarmer/memes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "      <td>https://huggingface.co/datasets/Splend1dchan/N...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>splend1dchan/nmsqa_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "      <td>https://huggingface.co/datasets/chamisfum/brai...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503185</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:28:15+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>robotis/ffw_bg2_rev4_pick_coffee_bottle_env5_14</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503186</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:29:40+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>robotis/ffw_bg2_rev4_pick_coffee_bottle_env5_15</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503187</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_5</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:30:53+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>dongkkka/ffw_bg2_rev4_custom_0919_5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503188</th>\n",
+       "      <td>chenxing1234567890/eval_testZ1.2.1</td>\n",
+       "      <td>https://huggingface.co/datasets/chenxing123456...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, tutorial</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:11+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/chenxing1234567890__eval_testZ...</td>\n",
+       "      <td>231</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>chenxing1234567890/eval_testz1.2.1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503189</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_6</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:09+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>dongkkka/ffw_bg2_rev4_custom_0919_6</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>503190 rows × 13 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             dataset_id  \\\n",
+       "0       akjadhav/leandojo-lean4-formal-informal-strings   \n",
+       "1                                          aemska/stuhl   \n",
+       "2                                 Pogpotatofarmer/memes   \n",
+       "3          Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+       "4                       chamisfum/brain_tumor_3_classes   \n",
+       "...                                                 ...   \n",
+       "503185  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14   \n",
+       "503186  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15   \n",
+       "503187              Dongkkka/ffw_bg2_rev4_custom_0919_5   \n",
+       "503188               chenxing1234567890/eval_testZ1.2.1   \n",
+       "503189              Dongkkka/ffw_bg2_rev4_custom_0919_6   \n",
+       "\n",
+       "                                              dataset_url  downloads author  \\\n",
+       "0       https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+       "1            https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+       "2       https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+       "3       https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+       "4       https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+       "...                                                   ...        ...    ...   \n",
+       "503185  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503186  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503187  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "503188  https://huggingface.co/datasets/chenxing123456...          0   None   \n",
+       "503189  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "\n",
+       "           license                                   tags task_categories  \\\n",
+       "0             None                                   None            None   \n",
+       "1         openrail                                   None            None   \n",
+       "2               cc                                   None            None   \n",
+       "3             None                                   None            None   \n",
+       "4             None                                   None            None   \n",
+       "...            ...                                    ...             ...   \n",
+       "503185  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503186  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503187  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503188  apache-2.0                      LeRobot, tutorial        robotics   \n",
+       "503189  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "\n",
+       "                    last_modified                                     reason  \\\n",
+       "0       2024-01-30 07:40:02+00:00             No metadata and no description   \n",
+       "1       2022-11-11 14:12:36+00:00  Short description (char count=0, words=0)   \n",
+       "2       2022-07-15 21:11:34+00:00  Short description (char count=0, words=0)   \n",
+       "3                            None                        Failed to load card   \n",
+       "4                            None                        Failed to load card   \n",
+       "...                           ...                                        ...   \n",
+       "503185  2025-09-19 06:28:15+00:00                                       None   \n",
+       "503186  2025-09-19 06:29:40+00:00                                       None   \n",
+       "503187  2025-09-19 06:30:53+00:00                                       None   \n",
+       "503188  2025-09-19 06:34:11+00:00                                       None   \n",
+       "503189  2025-09-19 06:34:09+00:00                                       None   \n",
+       "\n",
+       "                                              readme_path  word_count  \\\n",
+       "0       dataset_readmes/akjadhav__leandojo-lean4-forma...           0   \n",
+       "1                 dataset_readmes/aemska__stuhl_README.md           0   \n",
+       "2        dataset_readmes/Pogpotatofarmer__memes_README.md           0   \n",
+       "3                                                    None           0   \n",
+       "4                                                    None           0   \n",
+       "...                                                   ...         ...   \n",
+       "503185  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503186  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503187  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "503188  dataset_readmes/chenxing1234567890__eval_testZ...         231   \n",
+       "503189  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "\n",
+       "       category                                _dataset_id_lower  \n",
+       "0       minimal  akjadhav/leandojo-lean4-formal-informal-strings  \n",
+       "1       minimal                                     aemska/stuhl  \n",
+       "2       minimal                            pogpotatofarmer/memes  \n",
+       "3       minimal     splend1dchan/nmsqa_sew-d-tiny-100k-ft-ls100h  \n",
+       "4       minimal                  chamisfum/brain_tumor_3_classes  \n",
+       "...         ...                                              ...  \n",
+       "503185     rich  robotis/ffw_bg2_rev4_pick_coffee_bottle_env5_14  \n",
+       "503186     rich  robotis/ffw_bg2_rev4_pick_coffee_bottle_env5_15  \n",
+       "503187     rich              dongkkka/ffw_bg2_rev4_custom_0919_5  \n",
+       "503188     rich               chenxing1234567890/eval_testz1.2.1  \n",
+       "503189     rich              dongkkka/ffw_bg2_rev4_custom_0919_6  \n",
+       "\n",
+       "[503190 rows x 13 columns]"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "2bc30fa7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(503190, 14)\n",
+      "                                        dataset_id  \\\n",
+      "0  akjadhav/leandojo-lean4-formal-informal-strings   \n",
+      "1                                     aemska/stuhl   \n",
+      "2                            Pogpotatofarmer/memes   \n",
+      "3     Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+      "4                  chamisfum/brain_tumor_3_classes   \n",
+      "\n",
+      "                                         dataset_url  downloads author  \\\n",
+      "0  https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+      "1       https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+      "2  https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+      "3  https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+      "4  https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+      "\n",
+      "    license  tags task_categories              last_modified  \\\n",
+      "0      None  None            None  2024-01-30 07:40:02+00:00   \n",
+      "1  openrail  None            None  2022-11-11 14:12:36+00:00   \n",
+      "2        cc  None            None  2022-07-15 21:11:34+00:00   \n",
+      "3      None  None            None                       None   \n",
+      "4      None  None            None                       None   \n",
+      "\n",
+      "                                      reason  \\\n",
+      "0             No metadata and no description   \n",
+      "1  Short description (char count=0, words=0)   \n",
+      "2  Short description (char count=0, words=0)   \n",
+      "3                        Failed to load card   \n",
+      "4                        Failed to load card   \n",
+      "\n",
+      "                                         readme_path  word_count category  \\\n",
+      "0  dataset_readmes/akjadhav__leandojo-lean4-forma...           0  minimal   \n",
+      "1            dataset_readmes/aemska__stuhl_README.md           0  minimal   \n",
+      "2   dataset_readmes/Pogpotatofarmer__memes_README.md           0  minimal   \n",
+      "3                                               None           0  minimal   \n",
+      "4                                               None           0  minimal   \n",
+      "\n",
+      "           field keyword  \n",
+      "0            NaN     NaN  \n",
+      "1            NaN     NaN  \n",
+      "2            NaN     NaN  \n",
+      "3            NaN     NaN  \n",
+      "4  life_sciences   brain  \n"
+     ]
+    }
+   ],
+   "source": [
+    "# Merge on lowercase columns to bring 'field' and 'keyword' from csv_df\n",
+    "merged_df = merged_df.merge(\n",
+    "    csv_df[[\"_id_lower\", \"field\", \"keyword\"]],\n",
+    "    left_on=\"_dataset_id_lower\",\n",
+    "    right_on=\"_id_lower\",\n",
+    "    how=\"left\"\n",
+    ")\n",
+    "\n",
+    "# Drop the helper columns\n",
+    "merged_df = merged_df.drop(columns=[\"_dataset_id_lower\", \"_id_lower\"])\n",
+    "\n",
+    "# Quick check\n",
+    "print(merged_df.shape)\n",
+    "print(merged_df.head())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "4b104aef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset_id</th>\n",
+       "      <th>dataset_url</th>\n",
+       "      <th>downloads</th>\n",
+       "      <th>author</th>\n",
+       "      <th>license</th>\n",
+       "      <th>tags</th>\n",
+       "      <th>task_categories</th>\n",
+       "      <th>last_modified</th>\n",
+       "      <th>reason</th>\n",
+       "      <th>readme_path</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>category</th>\n",
+       "      <th>field</th>\n",
+       "      <th>keyword</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "      <td>https://huggingface.co/datasets/akjadhav/leand...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2024-01-30 07:40:02+00:00</td>\n",
+       "      <td>No metadata and no description</td>\n",
+       "      <td>dataset_readmes/akjadhav__leandojo-lean4-forma...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "      <td>https://huggingface.co/datasets/aemska/stuhl</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>openrail</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-11-11 14:12:36+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/aemska__stuhl_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Pogpotatofarmer/memes</td>\n",
+       "      <td>https://huggingface.co/datasets/Pogpotatofarme...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>None</td>\n",
+       "      <td>cc</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>2022-07-15 21:11:34+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Pogpotatofarmer__memes_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "      <td>https://huggingface.co/datasets/Splend1dchan/N...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "      <td>https://huggingface.co/datasets/chamisfum/brai...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>None</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>brain</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503185</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:28:15+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503186</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:29:40+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503187</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_5</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:30:53+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503188</th>\n",
+       "      <td>chenxing1234567890/eval_testZ1.2.1</td>\n",
+       "      <td>https://huggingface.co/datasets/chenxing123456...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, tutorial</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:11+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/chenxing1234567890__eval_testZ...</td>\n",
+       "      <td>231</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503189</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_6</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:09+00:00</td>\n",
+       "      <td>None</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>503190 rows × 14 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             dataset_id  \\\n",
+       "0       akjadhav/leandojo-lean4-formal-informal-strings   \n",
+       "1                                          aemska/stuhl   \n",
+       "2                                 Pogpotatofarmer/memes   \n",
+       "3          Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+       "4                       chamisfum/brain_tumor_3_classes   \n",
+       "...                                                 ...   \n",
+       "503185  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14   \n",
+       "503186  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15   \n",
+       "503187              Dongkkka/ffw_bg2_rev4_custom_0919_5   \n",
+       "503188               chenxing1234567890/eval_testZ1.2.1   \n",
+       "503189              Dongkkka/ffw_bg2_rev4_custom_0919_6   \n",
+       "\n",
+       "                                              dataset_url  downloads author  \\\n",
+       "0       https://huggingface.co/datasets/akjadhav/leand...         22   None   \n",
+       "1            https://huggingface.co/datasets/aemska/stuhl         11   None   \n",
+       "2       https://huggingface.co/datasets/Pogpotatofarme...         15   None   \n",
+       "3       https://huggingface.co/datasets/Splend1dchan/N...         11   None   \n",
+       "4       https://huggingface.co/datasets/chamisfum/brai...          8   None   \n",
+       "...                                                   ...        ...    ...   \n",
+       "503185  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503186  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0   None   \n",
+       "503187  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "503188  https://huggingface.co/datasets/chenxing123456...          0   None   \n",
+       "503189  https://huggingface.co/datasets/Dongkkka/ffw_b...          0   None   \n",
+       "\n",
+       "           license                                   tags task_categories  \\\n",
+       "0             None                                   None            None   \n",
+       "1         openrail                                   None            None   \n",
+       "2               cc                                   None            None   \n",
+       "3             None                                   None            None   \n",
+       "4             None                                   None            None   \n",
+       "...            ...                                    ...             ...   \n",
+       "503185  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503186  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503187  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503188  apache-2.0                      LeRobot, tutorial        robotics   \n",
+       "503189  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "\n",
+       "                    last_modified                                     reason  \\\n",
+       "0       2024-01-30 07:40:02+00:00             No metadata and no description   \n",
+       "1       2022-11-11 14:12:36+00:00  Short description (char count=0, words=0)   \n",
+       "2       2022-07-15 21:11:34+00:00  Short description (char count=0, words=0)   \n",
+       "3                            None                        Failed to load card   \n",
+       "4                            None                        Failed to load card   \n",
+       "...                           ...                                        ...   \n",
+       "503185  2025-09-19 06:28:15+00:00                                       None   \n",
+       "503186  2025-09-19 06:29:40+00:00                                       None   \n",
+       "503187  2025-09-19 06:30:53+00:00                                       None   \n",
+       "503188  2025-09-19 06:34:11+00:00                                       None   \n",
+       "503189  2025-09-19 06:34:09+00:00                                       None   \n",
+       "\n",
+       "                                              readme_path  word_count  \\\n",
+       "0       dataset_readmes/akjadhav__leandojo-lean4-forma...           0   \n",
+       "1                 dataset_readmes/aemska__stuhl_README.md           0   \n",
+       "2        dataset_readmes/Pogpotatofarmer__memes_README.md           0   \n",
+       "3                                                    None           0   \n",
+       "4                                                    None           0   \n",
+       "...                                                   ...         ...   \n",
+       "503185  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503186  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503187  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "503188  dataset_readmes/chenxing1234567890__eval_testZ...         231   \n",
+       "503189  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "\n",
+       "       category          field keyword  \n",
+       "0       minimal            NaN     NaN  \n",
+       "1       minimal            NaN     NaN  \n",
+       "2       minimal            NaN     NaN  \n",
+       "3       minimal            NaN     NaN  \n",
+       "4       minimal  life_sciences   brain  \n",
+       "...         ...            ...     ...  \n",
+       "503185     rich            NaN     NaN  \n",
+       "503186     rich            NaN     NaN  \n",
+       "503187     rich            NaN     NaN  \n",
+       "503188     rich            NaN     NaN  \n",
+       "503189     rich            NaN     NaN  \n",
+       "\n",
+       "[503190 rows x 14 columns]"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "69ec9289",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of rows with a value in 'science' column: 4040\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "# Replace all None with np.nan\n",
+    "merged_df = merged_df.replace({None: np.nan})\n",
+    "\n",
+    "# Count rows where 'science' column has a value (not NaN)\n",
+    "science_count = merged_df[\"field\"].notna().sum()\n",
+    "\n",
+    "print(f\"Number of rows with a value in 'science' column: {science_count}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "b0d58ceb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>dataset_id</th>\n",
+       "      <th>dataset_url</th>\n",
+       "      <th>downloads</th>\n",
+       "      <th>author</th>\n",
+       "      <th>license</th>\n",
+       "      <th>tags</th>\n",
+       "      <th>task_categories</th>\n",
+       "      <th>last_modified</th>\n",
+       "      <th>reason</th>\n",
+       "      <th>readme_path</th>\n",
+       "      <th>word_count</th>\n",
+       "      <th>category</th>\n",
+       "      <th>field</th>\n",
+       "      <th>keyword</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>akjadhav/leandojo-lean4-formal-informal-strings</td>\n",
+       "      <td>https://huggingface.co/datasets/akjadhav/leand...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-01-30 07:40:02+00:00</td>\n",
+       "      <td>No metadata and no description</td>\n",
+       "      <td>dataset_readmes/akjadhav__leandojo-lean4-forma...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>aemska/stuhl</td>\n",
+       "      <td>https://huggingface.co/datasets/aemska/stuhl</td>\n",
+       "      <td>11</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>openrail</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2022-11-11 14:12:36+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/aemska__stuhl_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Pogpotatofarmer/memes</td>\n",
+       "      <td>https://huggingface.co/datasets/Pogpotatofarme...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>cc</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2022-07-15 21:11:34+00:00</td>\n",
+       "      <td>Short description (char count=0, words=0)</td>\n",
+       "      <td>dataset_readmes/Pogpotatofarmer__memes_README.md</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h</td>\n",
+       "      <td>https://huggingface.co/datasets/Splend1dchan/N...</td>\n",
+       "      <td>11</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>chamisfum/brain_tumor_3_classes</td>\n",
+       "      <td>https://huggingface.co/datasets/chamisfum/brai...</td>\n",
+       "      <td>8</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Failed to load card</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0</td>\n",
+       "      <td>minimal</td>\n",
+       "      <td>life_sciences</td>\n",
+       "      <td>brain</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503185</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:28:15+00:00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503186</th>\n",
+       "      <td>ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15</td>\n",
+       "      <td>https://huggingface.co/datasets/ROBOTIS/ffw_bg...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:29:40+00:00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503187</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_5</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:30:53+00:00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503188</th>\n",
+       "      <td>chenxing1234567890/eval_testZ1.2.1</td>\n",
+       "      <td>https://huggingface.co/datasets/chenxing123456...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, tutorial</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:11+00:00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>dataset_readmes/chenxing1234567890__eval_testZ...</td>\n",
+       "      <td>231</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>503189</th>\n",
+       "      <td>Dongkkka/ffw_bg2_rev4_custom_0919_6</td>\n",
+       "      <td>https://huggingface.co/datasets/Dongkkka/ffw_b...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>apache-2.0</td>\n",
+       "      <td>LeRobot, ffw_bg2_rev4_custom, robotis</td>\n",
+       "      <td>robotics</td>\n",
+       "      <td>2025-09-19 06:34:09+00:00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...</td>\n",
+       "      <td>299</td>\n",
+       "      <td>rich</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>503190 rows × 14 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             dataset_id  \\\n",
+       "0       akjadhav/leandojo-lean4-formal-informal-strings   \n",
+       "1                                          aemska/stuhl   \n",
+       "2                                 Pogpotatofarmer/memes   \n",
+       "3          Splend1dchan/NMSQA_sew-d-tiny-100k-ft-ls100h   \n",
+       "4                       chamisfum/brain_tumor_3_classes   \n",
+       "...                                                 ...   \n",
+       "503185  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_14   \n",
+       "503186  ROBOTIS/ffw_bg2_rev4_pick_coffee_bottle_env5_15   \n",
+       "503187              Dongkkka/ffw_bg2_rev4_custom_0919_5   \n",
+       "503188               chenxing1234567890/eval_testZ1.2.1   \n",
+       "503189              Dongkkka/ffw_bg2_rev4_custom_0919_6   \n",
+       "\n",
+       "                                              dataset_url  downloads author  \\\n",
+       "0       https://huggingface.co/datasets/akjadhav/leand...         22    NaN   \n",
+       "1            https://huggingface.co/datasets/aemska/stuhl         11    NaN   \n",
+       "2       https://huggingface.co/datasets/Pogpotatofarme...         15    NaN   \n",
+       "3       https://huggingface.co/datasets/Splend1dchan/N...         11    NaN   \n",
+       "4       https://huggingface.co/datasets/chamisfum/brai...          8    NaN   \n",
+       "...                                                   ...        ...    ...   \n",
+       "503185  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0    NaN   \n",
+       "503186  https://huggingface.co/datasets/ROBOTIS/ffw_bg...          0    NaN   \n",
+       "503187  https://huggingface.co/datasets/Dongkkka/ffw_b...          0    NaN   \n",
+       "503188  https://huggingface.co/datasets/chenxing123456...          0    NaN   \n",
+       "503189  https://huggingface.co/datasets/Dongkkka/ffw_b...          0    NaN   \n",
+       "\n",
+       "           license                                   tags task_categories  \\\n",
+       "0              NaN                                    NaN             NaN   \n",
+       "1         openrail                                    NaN             NaN   \n",
+       "2               cc                                    NaN             NaN   \n",
+       "3              NaN                                    NaN             NaN   \n",
+       "4              NaN                                    NaN             NaN   \n",
+       "...            ...                                    ...             ...   \n",
+       "503185  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503186  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503187  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "503188  apache-2.0                      LeRobot, tutorial        robotics   \n",
+       "503189  apache-2.0  LeRobot, ffw_bg2_rev4_custom, robotis        robotics   \n",
+       "\n",
+       "                    last_modified                                     reason  \\\n",
+       "0       2024-01-30 07:40:02+00:00             No metadata and no description   \n",
+       "1       2022-11-11 14:12:36+00:00  Short description (char count=0, words=0)   \n",
+       "2       2022-07-15 21:11:34+00:00  Short description (char count=0, words=0)   \n",
+       "3                             NaN                        Failed to load card   \n",
+       "4                             NaN                        Failed to load card   \n",
+       "...                           ...                                        ...   \n",
+       "503185  2025-09-19 06:28:15+00:00                                        NaN   \n",
+       "503186  2025-09-19 06:29:40+00:00                                        NaN   \n",
+       "503187  2025-09-19 06:30:53+00:00                                        NaN   \n",
+       "503188  2025-09-19 06:34:11+00:00                                        NaN   \n",
+       "503189  2025-09-19 06:34:09+00:00                                        NaN   \n",
+       "\n",
+       "                                              readme_path  word_count  \\\n",
+       "0       dataset_readmes/akjadhav__leandojo-lean4-forma...           0   \n",
+       "1                 dataset_readmes/aemska__stuhl_README.md           0   \n",
+       "2        dataset_readmes/Pogpotatofarmer__memes_README.md           0   \n",
+       "3                                                     NaN           0   \n",
+       "4                                                     NaN           0   \n",
+       "...                                                   ...         ...   \n",
+       "503185  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503186  dataset_readmes/ROBOTIS__ffw_bg2_rev4_pick_cof...         299   \n",
+       "503187  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "503188  dataset_readmes/chenxing1234567890__eval_testZ...         231   \n",
+       "503189  dataset_readmes/Dongkkka__ffw_bg2_rev4_custom_...         299   \n",
+       "\n",
+       "       category          field keyword  \n",
+       "0       minimal            NaN     NaN  \n",
+       "1       minimal            NaN     NaN  \n",
+       "2       minimal            NaN     NaN  \n",
+       "3       minimal            NaN     NaN  \n",
+       "4       minimal  life_sciences   brain  \n",
+       "...         ...            ...     ...  \n",
+       "503185     rich            NaN     NaN  \n",
+       "503186     rich            NaN     NaN  \n",
+       "503187     rich            NaN     NaN  \n",
+       "503188     rich            NaN     NaN  \n",
+       "503189     rich            NaN     NaN  \n",
+       "\n",
+       "[503190 rows x 14 columns]"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "d8d61dc6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "merged_df saved to 'datasetcards.parquet'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Save to parquet\n",
+    "merged_df.to_parquet(\"datasetcards.parquet\", engine=\"pyarrow\", index=False)\n",
+    "\n",
+    "print(\"merged_df saved to 'datasetcards.parquet'\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hftest",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}