Update app.py

app.py

@@ -1,4 +1,4 @@
-import os, json, importlib.util, tempfile, traceback, torch
+import os, json, importlib.util, tempfile, traceback, torch, re, math
 import torch.nn.functional as F
 import gradio as gr
 import pandas as pd
@@ -54,27 +54,38 @@ def load_model(model_name: str):
 def _format_pct(x: float) -> str:
     return f"{x*100:.2f}%"
 
-def _predict_batch(texts, model_name, batch_size=64):
-    """รับ list[str] → คืน list[dict] = review, negative(%), positive(%), label"""
-    model, tok, cfg = load_model(model_name)
-    results = []
-    rows = [str(t) for t in texts if str(t).strip()]
-    for i in range(0, len(rows), batch_size):
-        chunk = rows[i:i+batch_size]
-        enc = tok(chunk, padding=True, truncation=True, max_length=cfg["max_len"], return_tensors="pt")
-        with torch.no_grad():
-            logits = model(enc["input_ids"], enc["attention_mask"])
-            probs = F.softmax(logits, dim=1).cpu().numpy()
-        for txt, p in zip(chunk, probs):
-            neg, pos = float(p[0]), float(p[1])
-            label = "positive" if pos >= neg else "negative"
-            results.append({
-                "review": txt,
-                "negative(%)": _format_pct(neg),
-                "positive(%)": _format_pct(pos),
-                "label": label,
-            })
-    return results
+# ====== ฟิลเตอร์ข้อความที่ไม่ใช่รีวิว / ค่าว่าง / สัญลักษณ์ ======
+_INVALID_STRINGS = {"-", "--", "—", "n/a", "na", "null", "none", "nan", ".", "…", ""}  # lower-case
+_RE_HAS_LETTER = re.compile(r"[ก-๙A-Za-z]")  # ต้องมีอย่างน้อย 1 ตัวอักษรไทยหรืออังกฤษ
+
+def _norm_text(v) -> str:
+    """แปลงค่าให้เป็นสตริงพร้อม trim และกัน NaN/None"""
+    if v is None:
+        return ""
+    if isinstance(v, float) and math.isnan(v):
+        return ""
+    s = str(v).strip()
+    return s
+
+def _is_substantive_text(s: str, min_chars: int = 2) -> bool:
+    """เงื่อนไขว่าเป็นข้อความที่พอจะวิเคราะห์ได้"""
+    if not s:
+        return False
+    s_lower = s.lower()
+    if s_lower in _INVALID_STRINGS:
+        return False
+    if not _RE_HAS_LETTER.search(s):
+        return False
+    if len(s.replace(" ", "")) < min_chars:
+        return False
+    return True
+
+def _clean_texts(texts):
+    """รับ list ใด ๆ → คืน (รายการที่ใช้ได้, จำนวนที่ถูกข้าม)"""
+    all_norm = [_norm_text(t) for t in texts]
+    cleaned = [t for t in all_norm if _is_substantive_text(t)]
+    skipped = len(all_norm) - len(cleaned)
+    return cleaned, skipped
 
 def _detect_cols(df: pd.DataFrame):
     """เดาชื่อคอลัมน์รีวิว/ร้านอัตโนมัติ ถ้าไม่พบรีวิว เลือกคอลัมน์ object ตัวแรก"""
@@ -172,13 +183,36 @@ def _shop_summary(out_df: pd.DataFrame, max_shops=15):
     )
     return fig, table
 
+# ---------- core prediction ----------
+def _predict_batch(texts, model_name, batch_size=64):
+    """รับ list[str] (ผ่านการกรองแล้ว) → คืน list[dict]"""
+    model, tok, cfg = load_model(model_name)
+    results = []
+    for i in range(0, len(texts), batch_size):
+        chunk = texts[i:i+batch_size]
+        enc = tok(chunk, padding=True, truncation=True, max_length=cfg["max_len"], return_tensors="pt")
+        with torch.no_grad():
+            logits = model(enc["input_ids"], enc["attention_mask"])
+            probs = F.softmax(logits, dim=1).cpu().numpy()
+        for txt, p in zip(chunk, probs):
+            neg, pos = float(p[0]), float(p[1])
+            label = "positive" if pos >= neg else "negative"
+            results.append({
+                "review": txt,
+                "negative(%)": _format_pct(neg),
+                "positive(%)": _format_pct(pos),
+                "label": label,
+            })
+    return results
+
 # ---------- API wrappers ----------
 def predict_one(text: str, model_choice: str):
     try:
-        if not text.strip():
-            return {"negative": 0.0, "positive": 0.0}, ""
+        s = _norm_text(text)
+        if not _is_substantive_text(s):
+            return {"negative": 0.0, "positive": 0.0}, "invalid"
         model_name = "baseline" if model_choice == "baseline" else "cnn_bilstm"
-        out = _predict_batch([text], model_name)[0]
+        out = _predict_batch([s], model_name)[0]
         probs = {
             "negative": float(out["negative(%)"].rstrip("%"))/100.0,
             "positive": float(out["positive(%)"].rstrip("%"))/100.0,
@@ -192,12 +226,19 @@ def predict_one(text: str, model_choice: str):
 def predict_many(text_block: str, model_choice: str):
     try:
         model_name = "baseline" if model_choice == "baseline" else "cnn_bilstm"
-        lines = [ln.strip() for ln in (text_block or "").splitlines() if ln.strip()]
-        results = _predict_batch(lines, model_name)
+        raw_lines = (text_block or "").splitlines()
+        trimmed = [_norm_text(ln) for ln in raw_lines if _norm_text(ln)]
+        cleaned, skipped = _clean_texts(trimmed)
+
+        if len(cleaned) == 0:
+            empty = pd.DataFrame(columns=["review","negative(%)","positive(%)","label"])
+            return empty, go.Figure(), go.Figure(), "No valid text"
+
+        results = _predict_batch(cleaned, model_name)
         df = pd.DataFrame(results, columns=["review","negative(%)","positive(%)","label"])
-        if len(df) == 0:
-            return df, go.Figure(), go.Figure(), "No data"
+
         fig_bar, fig_pie, info_md = _make_figures(df)
+        info_md = f"{info_md}  \n- Skipped (empty/non-text): {skipped}"
         return df, fig_bar, fig_pie, info_md
     except Exception as e:
         print("ERROR in predict_many:", repr(e))
@@ -219,11 +260,21 @@ def predict_csv(file_obj, model_choice: str, review_col_override: str = "", shop
         if rev_col not in df.columns:
             raise ValueError(f"ไม่พบคอลัมน์รีวิว '{rev_col}' ใน CSV (columns = {list(df.columns)})")
 
-        results = _predict_batch(df[rev_col].astype(str).tolist(), model_name)
+        # === กรองแถวที่ใช้ได้จริง ===
+        reviews_norm = df[rev_col].apply(_norm_text)
+        mask_use = reviews_norm.apply(_is_substantive_text)
+        skipped = int((~mask_use).sum())
+
+        used_df = df.loc[mask_use].copy()
+        if used_df.empty:
+            empty = pd.DataFrame(columns=["review","negative(%)","positive(%)","label"])
+            return empty, None, go.Figure(), go.Figure(), go.Figure(), pd.DataFrame(), "ไม่พบรีวิวที่เป็นข้อความ"
+
+        results = _predict_batch(used_df[rev_col].astype(str).tolist(), model_name)
         out = pd.DataFrame(results, columns=["review","negative(%)","positive(%)","label"])
 
-        if shop_col and shop_col in df.columns:
-            out.insert(0, "shop", df[shop_col].astype(str).fillna(""))
+        if shop_col and shop_col in used_df.columns:
+            out.insert(0, "shop", used_df[shop_col].astype(str).fillna(""))
 
         # ไฟล์ผลลัพธ์สำหรับดาวน์โหลด
         tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv")
@@ -234,8 +285,13 @@ def predict_csv(file_obj, model_choice: str, review_col_override: str = "", shop
         # กราฟ/ตารางต่อร้าน (ถ้ามี shop)
         fig_shop, tbl_shop = _shop_summary(out)
 
-        # แนบข้อความบอกคอลัมน์ที่ใช้
-        info_md = f"{info_md}  \nใช้คอลัมน์รีวิว: {rev_col}" + (f" | คอลัมน์ร้าน: {shop_col}" if ("shop" in out.columns) else " | ไม่มีคอลัมน์ร้าน")
+        # แนบข้อความบอกคอลัมน์ที่ใช้ + จำนวนแถวที่ถูกข้าม
+        info_md = (
+            f"{info_md}  \n"
+            f"ใช้คอลัมน์รีวิว: {rev_col}"
+            + (f" | คอลัมน์ร้าน: {shop_col}" if ("shop" in out.columns) else " | ไม่มีคอลัมน์ร้าน")
+            + f"  \n- Skipped (empty/non-text): {skipped}"
+        )
 
         return out, tmp.name, fig_bar, fig_pie, fig_shop, tbl_shop, info_md
     except Exception as e: