clean up design

app.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import re
 from datatrove.pipeline.readers import ParquetReader
 from default_wiki_pipeline import _parse_and_clean_wikicode, mwparserfromhell
 
@@ -23,7 +24,15 @@ def _build_header_markdown(doc) -> str:
         header += f"\n\n[{url}]({url})"
     return header
 
-def matches_filters(doc, require_has_math: bool | None, require_has_infobox: bool | None) -> bool:
+def has_markdown_table(md_text: str) -> bool:
+    return bool(re.search(r"(?m)^\s*\|.+\|\s*\n\s*\|?\s*:?-{3,}.*$", md_text or ""))
+
+
+def has_code_fence(md_text: str) -> bool:
+    return "```" in (md_text or "")
+
+
+def matches_prefilters(doc, require_has_math: bool | None, require_has_infobox: bool | None) -> bool:
     meta = doc.metadata or {}
     if require_has_math and not bool(meta.get("has_math")):
         return False
@@ -32,22 +41,37 @@ def matches_filters(doc, require_has_math: bool | None, require_has_infobox: boo
     return True
 
 
-def find_next_matching_from(docs_cache, reader_iter, start_idx: int, require_has_math: bool | None, require_has_infobox: bool | None):
-    # Scan cache first
+def postfilters_ok(md_text: str, require_has_table: bool | None, require_has_code: bool | None) -> bool:
+    if require_has_table and not has_markdown_table(md_text):
+        return False
+    if require_has_code and not has_code_fence(md_text):
+        return False
+    return True
+
+
+def find_next_valid(docs_cache, reader_iter, start_idx: int, require_has_math: bool | None, require_has_infobox: bool | None, require_has_table: bool | None, require_has_code: bool | None):
+    # Scan cache first (forward from start_idx)
     i = max(-1, start_idx)
     while i + 1 < len(docs_cache):
         i += 1
-        if matches_filters(docs_cache[i], require_has_math, require_has_infobox):
-            return i, docs_cache, reader_iter
+        if not matches_prefilters(docs_cache[i], require_has_math, require_has_infobox):
+            continue
+        left, left_meta, md, info, right, header = render_idx(docs_cache, i)
+        if postfilters_ok(left, require_has_table, require_has_code):
+            return i, docs_cache, reader_iter, left, md, left_meta, header, md, info, right
     # Stream until found or exhausted
     while True:
         prev_len = len(docs_cache)
         docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, prev_len)
         if len(docs_cache) == prev_len:
             break
-        if matches_filters(docs_cache[-1], require_has_math, require_has_infobox):
-            return len(docs_cache) - 1, docs_cache, reader_iter
-    return -1, docs_cache, reader_iter
+        if not matches_prefilters(docs_cache[-1], require_has_math, require_has_infobox):
+            continue
+        left, left_meta, md, info, right, header = render_idx(docs_cache, len(docs_cache) - 1)
+        if postfilters_ok(left, require_has_table, require_has_code):
+            idx = len(docs_cache) - 1
+            return idx, docs_cache, reader_iter, left, md, left_meta, header, md, info, right
+    return -1, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("")
 
 def render_iframe(url: str, height: int = 800) -> str:
     safe_url = url or "about:blank"
@@ -87,7 +111,7 @@ def _ensure_until_index(docs_cache, reader_iter, target_idx: int):
     return docs_cache, reader_iter
 
 
-def on_select_language(lang: str, require_has_math: bool, require_has_infobox: bool):
+def on_select_language(lang: str, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool):
     """Load documents for the selected language from HF Parquet and display."""
     language = (lang or "").strip()
     if not language:
@@ -101,81 +125,16 @@ def on_select_language(lang: str, require_has_math: bool, require_has_infobox: b
     docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
     if not docs_cache:
         return (-1, [], reader_iter, "No documents found.", {}, "", [], render_iframe(""))
-    # Find first doc matching filters (starting before 0)
-    idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, -1, require_has_math, require_has_infobox)
+    # Find first doc matching pre- and post-filters
+    idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right = find_next_valid(
+        docs_cache, reader_iter, -1, require_has_math, require_has_infobox, require_has_table, require_has_code
+    )
     if idx == -1:
-        return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-    left, left_meta, md, info, right, header = render_idx(docs_cache, idx)
-    return (idx, docs_cache, reader_iter, left, left, left_meta, header, md, info, right)
+        return (-1, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe(""))
+    return (idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right)
 
 
-def on_find(docs_cache, idx: int, reader_iter, id_query: str, require_has_math: bool, require_has_infobox: bool):
-    query = (id_query or "").strip()
-    if not docs_cache and reader_iter is None:
-        return -1, docs_cache, reader_iter, "No documents loaded.", {}, "", [], render_iframe("")
-    if not query:
-        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
-        new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, -1, require_has_math, require_has_infobox)
-        if new_idx == -1:
-            return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-        left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-        return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
-    # Exact match in cache
-    for i, doc in enumerate(docs_cache):
-        meta = (getattr(doc, "metadata", None) or {})
-        doc_id = (getattr(doc, "id", None) or "")
-        url = meta.get("url") or ""
-        if doc_id == query or meta.get("wikidata_id") == query or url == query:
-            left, left_meta, md, info, right, header = render_idx(docs_cache, i)
-            if matches_filters(doc, require_has_math, require_has_infobox):
-                return i, docs_cache, reader_iter, left, left_meta, header, md, info, right
-            new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, i, require_has_math, require_has_infobox)
-            if new_idx == -1:
-                return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-            left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-            return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
-    # Suffix match in cache
-    for i, doc in enumerate(docs_cache):
-        doc_id = (getattr(doc, "id", None) or "")
-        meta = (getattr(doc, "metadata", None) or {})
-        url = meta.get("url") or ""
-        if doc_id.endswith(f"/{query}") or url.endswith(query):
-            left, left_meta, md, info, right, header = render_idx(docs_cache, i)
-            if matches_filters(doc, require_has_math, require_has_infobox):
-                return i, docs_cache, reader_iter, left, left_meta, header, md, info, right
-            new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, i, require_has_math, require_has_infobox)
-            if new_idx == -1:
-                return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-            left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-            return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
-    # Stream forward until found or exhausted
-    found_idx = None
-    while True:
-        prev_len = len(docs_cache)
-        docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, prev_len)
-        if len(docs_cache) == prev_len:
-            break
-        doc = docs_cache[-1]
-        meta = (getattr(doc, "metadata", None) or {})
-        doc_id = (getattr(doc, "id", None) or "")
-        url = meta.get("url") or ""
-        if doc_id == query or meta.get("wikidata_id") == query or url.endswith(query) or url == query or doc_id.endswith(f"/{query}"):
-            found_idx = len(docs_cache) - 1
-            break
-    if found_idx is not None:
-        new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, found_idx - 1, require_has_math, require_has_infobox)
-        if new_idx == -1:
-            return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-        left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-        return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
-    target_idx = 0 if docs_cache else -1
-    if target_idx == -1:
-        return -1, docs_cache, reader_iter, "No documents found.", {}, "", [], render_iframe("")
-    new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, target_idx - 1, require_has_math, require_has_infobox)
-    if new_idx == -1:
-        return (-1, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe(""))
-    left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-    return new_idx, docs_cache, reader_iter, left, left_meta, header, md, info, right
+    # on_find removed per user request
 
 
 def show_doc(doc):
@@ -199,49 +158,38 @@ def render_idx(docs, idx: int):
     return left, left_meta, md, info, right, header
 
 
-def on_prev(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool):
+def on_prev(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool):
     if not docs_cache:
         # Try to ensure at least first doc is loaded
         docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, 0)
         if not docs_cache:
             return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("")
     new_idx = max(0, idx - 1)
-    # Apply filters going backwards by scanning from start to new_idx
-    filtered_idx = new_idx
-    if new_idx >= 0:
-        for i in range(new_idx, -1, -1):
-            if matches_filters(docs_cache[i], require_has_math, require_has_infobox):
-                filtered_idx = i
-                break
-    left, left_meta, md, info, right, header = render_idx(docs_cache, filtered_idx)
-    return filtered_idx, docs_cache, reader_iter, left, left, left_meta, header, md, info, right
-
-
-def on_next(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool):
+    # Apply prefilters going backwards by scanning from start to new_idx; evaluate postfilters on candidate
+    filtered_idx = -1
+    for i in range(new_idx, -1, -1):
+        if not matches_prefilters(docs_cache[i], require_has_math, require_has_infobox):
+            continue
+        left, left_meta, md, info, right, header = render_idx(docs_cache, i)
+        if postfilters_ok(md, require_has_table, require_has_code):
+            filtered_idx = i
+            return filtered_idx, docs_cache, reader_iter, left, md, left_meta, header, md, info, right
+    return idx, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("")
+
+
+def on_next(docs_cache, idx: int, reader_iter, require_has_math: bool, require_has_infobox: bool, require_has_table: bool, require_has_code: bool):
     target_idx = idx + 1 if idx >= 0 else 0
     docs_cache, reader_iter = _ensure_until_index(docs_cache, reader_iter, target_idx)
     if not docs_cache:
         return idx, docs_cache, reader_iter, "No documents.", {}, "", [], render_iframe("")
-    new_idx = min(len(docs_cache) - 1, target_idx)
-    # Apply filters forward
-    new_idx, docs_cache, reader_iter = find_next_matching_from(docs_cache, reader_iter, idx, require_has_math, require_has_infobox)
+    # Apply filters forward using new finder
+    new_idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right = find_next_valid(
+        docs_cache, reader_iter, idx, require_has_math, require_has_infobox, require_has_table, require_has_code
+    )
     if new_idx == -1:
-        return idx, docs_cache, reader_iter, "No documents match filters.", {}, "", [], render_iframe("")
-    left, left_meta, md, info, right, header = render_idx(docs_cache, new_idx)
-    return new_idx, docs_cache, reader_iter, left, left, left_meta, header, md, info, right
-
-
-SCROLL_TO_TOP_JS = """
-() => {
-  const ids = ["left_text_box", "right_markdown_box"]; 
-  for (const id of ids) {
-    const root = document.getElementById(id);
-    if (!root) continue;
-    const ta = root.querySelector('textarea');
-    if (ta) ta.scrollTop = 0;
-  }
-}
-"""
+        return idx, docs_cache, reader_iter, "No documents match filters.", "", {}, "", "", [], render_iframe("")
+    return new_idx, docs_cache, reader_iter, left, md, left_meta, header, right_md, info, right
+
 
 with gr.Blocks() as demo:
     idx_state = gr.State(value=-1, time_to_live=900)
@@ -264,6 +212,8 @@ with gr.Blocks() as demo:
                 with gr.Column(scale=1):
                     require_has_math = gr.Checkbox(label="Has math", value=False)
                     require_has_infobox = gr.Checkbox(label="Has infobox", value=False)
+                    require_has_table = gr.Checkbox(label="Has table", value=False)
+                    require_has_code = gr.Checkbox(label="Has pre/code", value=False)
     with gr.Row():
         with gr.Column():
             with gr.Tab("FineWiki raw"):
@@ -285,21 +235,15 @@ with gr.Blocks() as demo:
                 right_markdown = gr.Textbox(label="wikimedia/wikipedia extraction", lines=30, elem_id="right_markdown_box")
 
 
-    _ev1 = language_select.change(on_select_language, inputs=[language_select, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev1.then(js=SCROLL_TO_TOP_JS)
-    _ev2 = demo.load(on_select_language, inputs=[language_select, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev2.then(js=SCROLL_TO_TOP_JS)
+language_select.change(on_select_language, inputs=[language_select, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
+demo.load(on_select_language, inputs=[language_select, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
     # find_btn.click(on_find, inputs=[docs_state, idx_state, iter_state, id_input, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
 
     # Visibility toggles driven directly by checkbox changes
-    _ev4 = prev_btn.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev4.then(js=SCROLL_TO_TOP_JS)
-    _ev5 = next_btn.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev5.then(js=SCROLL_TO_TOP_JS)
-    _ev4 = prev_btn2.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev4.then(js=SCROLL_TO_TOP_JS)
-    _ev5 = next_btn2.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
-    _ev5.then(js=SCROLL_TO_TOP_JS)
+prev_btn.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
+next_btn.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
+prev_btn2.click(on_prev, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
+next_btn2.click(on_next, inputs=[docs_state, idx_state, iter_state, require_has_math, require_has_infobox, require_has_table, require_has_code], outputs=[idx_state, docs_state, iter_state, left_text_raw, left_text_md, left_meta, header_md, right_markdown, right_infoboxes, right_iframe], concurrency_limit=1)
 
     # Enable global queue to coordinate concurrent requests safely
     demo.queue(default_concurrency_limit=1, max_size=128)