Multiple pace and \n charachters fix

app.py

@@ -2,6 +2,10 @@ import gradio as gr
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import torch
 import re
+from tokenizers import normalizers
+from tokenizers.normalizers import Sequence, Replace, Strip
+from tokenizers import Regex
+
 model1_path = "modernbert.bin"
 model2_path = "https://huggingface.co/mihalykiss/modernbert_2/resolve/main/Model_groups_3class_seed12"
 model3_path = "https://huggingface.co/mihalykiss/modernbert_2/resolve/main/Model_groups_3class_seed22"
@@ -35,22 +39,21 @@ label_mapping = {
     39: 'text-davinci-002', 40: 'text-davinci-003'
 }
 
-def clean_text(text):
-
-    text = text.replace("\r\n", "\n").replace("\r", "\n")
-    
- 
-    text = re.sub(r"\n\s*\n+", "\n\n", text)  
-    
-    text = re.sub(r"[ \t]+", " ", text)
+def clean_text(text: str) -> str:
+    text = re.sub(r'\s{2,}', ' ', text)                    
+    text = re.sub(r'\s+([,.;:?!])', r'\1', text)
+    return text
 
-    text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)  
 
-    text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)  
+newline_to_space  = Replace(Regex(r'\s*\n\s*'), " ")
+join_hyphen_break = Replace(Regex(r'(\w+)[--]\s*\n\s*(\w+)'), r"\1\2")
 
-    text = text.strip()
-    
-    return text
+tokenizer.backend_tokenizer.normalizer = Sequence([
+    tokenizer.backend_tokenizer.normalizer,
+    join_hyphen_break,
+    newline_to_space,
+    Strip()
+])
 
 def classify_text(text):
     cleaned_text = clean_text(text)
@@ -60,7 +63,7 @@ def classify_text(text):
         )
         return result_message
 
-    inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True).to(device)
+    inputs = tokenizer(cleaned_text, return_tensors="pt", truncation=True, padding=True).to(device)
 
     with torch.no_grad():
         logits_1 = model_1(**inputs).logits