Update utils.py

utils.py

@@ -1,4 +1,4 @@
-from transformers import AutoTokenizer
+from tokenizers import Tokenizer
 import logging
 import os
 
@@ -8,7 +8,7 @@ logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %
 def get_tokenizer(id):
     logging.debug(f"Loading tokenizer: {id}")
     try:
-        tokenizer = AutoTokenizer.from_pretrained(id, trust_remote_code=True, token=os.getenv("HF_TOKEN"))
+        tokenizer = Tokenizer.from_pretrained(id, trust_remote_code=True, token=os.getenv("HF_TOKEN"))
         logging.debug(f"Tokenizer loaded: {tokenizer}")
         return tokenizer
     except Exception as e: