Update requirements.txt

requirements.txt

@@ -3,14 +3,14 @@
 # Core compute (CUDA 12.x -> cu121 wheels)
 torch==2.3.1+cu121
 
-# NLP stack
-transformers==4.44.2        # ✅ compatible with tokenizers 0.20.x
-tokenizers==0.20.1          # ✅ parses Apertus tokenizer.json
+# HF stack (PAIR THESE VERSIONS)
+transformers==4.44.2        # requires tokenizers >=0.19,<0.20
+tokenizers==0.19.1          # ✅ compatible with 4.44.2
+sentencepiece==0.1.99
+huggingface_hub==0.24.6
 accelerate==0.33.0
 safetensors==0.4.4
 bitsandbytes==0.43.1
-sentencepiece==0.1.99
-huggingface_hub==0.24.6
 regex==2024.7.24
 
 # UI
@@ -21,4 +21,3 @@ websockets>=12.0
 # Data helpers
 pandas==2.2.2
 numpy==1.26.4
-