salmankhanpm commited on
Commit
0ee82fd
·
verified ·
1 Parent(s): 07d05e5

include custom telugu tokenizer expansion of gemma3 tokenizer

Browse files
Files changed (1) hide show
  1. app.py +2 -1
app.py CHANGED
@@ -22,7 +22,8 @@ def load_test_phrases(filename):
22
  return file.read().splitlines()
23
 
24
 
25
- models = ["HuggingFaceTB/SmolLM2-135M-Instruct", # SmolLM2
 
26
  "meta-llama/Llama-3.2-3B-Instruct", # LLAMA-3
27
  "Telugu-LLM-Labs/Telugu-Llama2-7B-v0-Instruct", # LLama 2 Finetuned for Improving Telugu
28
  "CohereForAI/aya-23-8B", # AYA
 
22
  return file.read().splitlines()
23
 
24
 
25
+ models = ["salmankhanpm/gemma3-tokenizer-telugu", # Gemma 3 Telugu Expansion
26
+ "HuggingFaceTB/SmolLM2-135M-Instruct", # SmolLM2
27
  "meta-llama/Llama-3.2-3B-Instruct", # LLAMA-3
28
  "Telugu-LLM-Labs/Telugu-Llama2-7B-v0-Instruct", # LLama 2 Finetuned for Improving Telugu
29
  "CohereForAI/aya-23-8B", # AYA