include custom telugu tokenizer expansion of gemma3 tokenizer
Browse files
app.py
CHANGED
|
@@ -22,7 +22,8 @@ def load_test_phrases(filename):
|
|
| 22 |
return file.read().splitlines()
|
| 23 |
|
| 24 |
|
| 25 |
-
models = ["
|
|
|
|
| 26 |
"meta-llama/Llama-3.2-3B-Instruct", # LLAMA-3
|
| 27 |
"Telugu-LLM-Labs/Telugu-Llama2-7B-v0-Instruct", # LLama 2 Finetuned for Improving Telugu
|
| 28 |
"CohereForAI/aya-23-8B", # AYA
|
|
|
|
| 22 |
return file.read().splitlines()
|
| 23 |
|
| 24 |
|
| 25 |
+
models = ["salmankhanpm/gemma3-tokenizer-telugu", # Gemma 3 Telugu Expansion
|
| 26 |
+
"HuggingFaceTB/SmolLM2-135M-Instruct", # SmolLM2
|
| 27 |
"meta-llama/Llama-3.2-3B-Instruct", # LLAMA-3
|
| 28 |
"Telugu-LLM-Labs/Telugu-Llama2-7B-v0-Instruct", # LLama 2 Finetuned for Improving Telugu
|
| 29 |
"CohereForAI/aya-23-8B", # AYA
|