Upload 2 files

_app.py

@@ -1,150 +1,83 @@
-import logging
-
 import tiktoken
 from transformers import AutoTokenizer
 
-import gradio as gr
-
-logger = logging.getLogger(__name__)  # noqa
-
-
-def load_test_phrases(filename):
-    with open(f"./data/{filename}", "r", encoding="utf-8") as file:
-        return file.read().splitlines()
-
-
-models = ["HuggingFaceTB/SmolLM2-135M-Instruct",     # SmolLM2 
-          "meta-llama/Llama-3.2-3B-Instruct",        # LLAMA-3
-          "Telugu-LLM-Labs/Telugu-Llama2-7B-v0-Instruct", # LLama 2 Finetuned for Improving Telugu
-          "CohereForAI/aya-23-8B",                   # AYA
-          "google/gemma-3-4b-it",                    # GEMMA 3
-          "sarvamai/sarvam-1",                       # SarvamAI   
-          "gpt-4o",                                  # GPT4o
-          "Qwen/Qwen3-4B",                           # Qwen
-          "TWO/sutra-mlt256-v2"]                     # SUTRA
-
-test_phrase_set = [
-    "ఐదు వాక్యాలలో న్యూట్రాన్ స్కాటరింగ్ గురించి నాకు వివరణ ఇవ్వండి",  # Telugu
-]
-
-test_phrase_set_long_1 = load_test_phrases('multilingualphrases01.txt')
-test_phrase_set_long_2 = load_test_phrases('multilingualphrases02.txt')
-test_phrase_set_long_3 = load_test_phrases('multilingualphrases03.txt')
-
-
-def generate_tokens_as_table(text):
+# ... existing code ...
+def analyze_tokens_detailed(text, model):
+    """
+    For a given text and model, returns a list of dicts with details for each token:
+    - token string
+    - token id
+    - decoded value
+    - token length
+    - NSL value (token length / max token length in sequence)
+    - subword fertility (number of tokens per word)
+    Also returns the decoded output for the entire sequence.
+    """
+    # Tokenize
+    if 'gpt' in model:
+        tokenizer = tiktoken.encoding_for_model(model)
+        token_ids = tokenizer.encode(text)
+        tokens = [tokenizer.decode([tid]) for tid in token_ids]
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(model)
+        token_ids = tokenizer.encode(text, add_special_tokens=False)
+        tokens = [tokenizer.decode([tid]) for tid in token_ids]
+
+    # Decoded output for the entire sequence
+    if 'gpt' in model:
+        decoded_output = tokenizer.decode(token_ids)
+    else:
+        decoded_output = tokenizer.decode(token_ids)
+
+    # Token lengths
+    token_lengths = [len(t) for t in tokens]
+    max_token_length = max(token_lengths) if token_lengths else 1
+    nsl_values = [l / max_token_length for l in token_lengths]
+
+    # Subword fertility: number of tokens per word
+    # Map each token to its originating word (approximate)
+    words = text.split()
+    word_token_counts = []
+    if len(words) > 0:
+        # Use a simple greedy approach: assign tokens to words in order
+        import re
+        text_pointer = 0
+        word_idx = 0
+        token_word_map = []
+        for token in tokens:
+            # Find the next word that matches the start of the token
+            while word_idx < len(words) and not text[text_pointer:].startswith(words[word_idx]):
+                text_pointer += 1
+            if word_idx < len(words):
+                token_word_map.append(word_idx)
+                text_pointer += len(token)
+                if text_pointer >= len(text) or (word_idx + 1 < len(words) and text[text_pointer:].startswith(words[word_idx + 1])):
+                    word_idx += 1
+            else:
+                token_word_map.append(-1)
+        # Count tokens per word
+        from collections import Counter
+        fertility_counter = Counter(token_word_map)
+        subword_fertility = [fertility_counter[i] for i in range(len(words))]
+        # Assign fertility to each token
+        token_fertility = [fertility_counter[idx] if idx >= 0 else 0 for idx in token_word_map]
+    else:
+        token_fertility = [1 for _ in tokens]
+
+    # Build table
     table = []
-    for model in models:
-        if 'gpt' not in model:
-            tokenizer = AutoTokenizer.from_pretrained(model)
-            tokens = tokenizer.encode(text, add_special_tokens=False)
-        else:
-            tokenizer = tiktoken.encoding_for_model(model)
-            tokens = tokenizer.encode(text)
-        decoded = [tokenizer.decode([t]) for t in tokens]
-        table.append([model] + decoded)
-    return table
-
-
-def generate_tokenizer_table(text):
-    if not text:
-        return []
-
-    token_counts = {model: 0 for model in models}
-    vocab_size = {model: 0 for model in models}
-
-    for model in models:
-        if 'gpt' not in model:
-            tokenizer = AutoTokenizer.from_pretrained(model)
-            vocab_size[model] = tokenizer.vocab_size
-        else:
-            tokenizer = tiktoken.encoding_for_model(model)
-            vocab_size[model] = tokenizer.n_vocab
-
-        token_counts[model] += len(tokenizer.encode(text))
-
-    word_count = len(text.split(' '))
-
-    output = []
-    for m in models:
-        row = [m, vocab_size[m], word_count, token_counts[m], f"{token_counts[m] / word_count:0.2f}"]
-        output.append(row)
-
-    return output
-
-
-def generate_split_token_table(text):
-    if not text:
-        return gr.Dataframe()
-
-    table = generate_tokenizer_table(text)
-    return gr.Dataframe(
-        table,
-        headers=['tokenizer', 'v size', '#word', '#token', '#tokens/word'],
-        datatype=["str", "number", "str"],
-        row_count=len(models),
-        col_count=(5, "fixed"),
-    )
-
-
-with gr.Blocks() as sutra_token_count:
-    gr.Markdown(
-        """
-        # SUTRA Multilingual Tokenizer Specs & Stats.
-        ## Tokenize paragraphs in multiple languages and compare token counts.
-        """)
-    textbox = gr.Textbox(label="Input Text")
-    submit_button = gr.Button("Submit")
-    output = gr.Dataframe()
-    examples = [
-        [' '.join(test_phrase_set_long_1)],
-        [' '.join(test_phrase_set_long_2)],
-        [' '.join(test_phrase_set_long_3)],
-    ]
-    gr.Examples(examples=examples, inputs=[textbox])
-    submit_button.click(generate_split_token_table, inputs=[textbox], outputs=[output])
-
-
-def generate_tokens_table(text):
-    table = generate_tokens_as_table(text)
-    cols = len(table[0])
-    return gr.Dataframe(
-        table,
-        headers=['model'] + [str(i) for i in range(cols - 1)],
-        row_count=2,
-        col_count=(cols, "fixed"),
-    )
-
-
-with gr.Blocks() as sutra_tokenize:
-    gr.Markdown(
-        """
-        # SUTRA Multilingual Tokenizer Sentence Inspector.
-        ## Tokenize a sentence with various tokenizers and inspect how it's broken down.
-        """)
-    textbox = gr.Textbox(label="Input Text")
-    submit_button = gr.Button("Submit")
-    output = gr.Dataframe()
-    examples = test_phrase_set
-    gr.Examples(examples=examples, inputs=[textbox])
-    submit_button.click(generate_tokens_table, inputs=[textbox], outputs=[output])
-
-
-if __name__ == '__main__':
-    with gr.Blocks(analytics_enabled=False) as demo:
-        with gr.Row():
-            gr.Markdown(
-                """
-                ## <img src="https://playground.two.ai/sutra.svg" height="20"/>
-                """
-            )
-        with gr.Row():
-            gr.TabbedInterface(
-                interface_list=[sutra_tokenize, sutra_token_count],
-                tab_names=["Tokenize Text", "Tokenize Paragraphs"]
-            )
-
-demo.queue(default_concurrency_limit=5).launch(
-    server_name="0.0.0.0",
-    allowed_paths=["/"],
-)
+    for i, (token, tid, decoded, length, nsl, fert) in enumerate(zip(tokens, token_ids, tokens, token_lengths, nsl_values, token_fertility)):
+        table.append({
+            'token': token,
+            'token_id': tid,
+            'decoded': decoded,
+            'token_length': length,
+            'nsl': nsl,
+            'subword_fertility': fert
+        })
+    return {
+        'model': model,
+        'decoded_output': decoded_output,
+        'tokens': table
+    }
+# ... existing code ...

requirements.txt

@@ -1,4 +1,71 @@
 transformers
 tiktoken
 gradio
-sentencepiece
+sentencepieceaiofiles==24.1.0
+annotated-types==0.7.0
+anyio==4.9.0
+brotli==1.1.0
+certifi==2025.7.14
+charset-normalizer==3.4.2
+click==8.2.1
+dotenv==0.9.9
+fastapi==0.116.1
+ffmpy==0.6.1
+filelock==3.18.0
+fsspec==2025.7.0
+gradio==5.38.2
+gradio-client==1.11.0
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.5
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.34.1
+idna==3.10
+inquirerpy==0.3.4
+jinja2==3.1.6
+markdown-it-py==3.0.0
+markupsafe==3.0.2
+mdurl==0.1.2
+numpy==2.3.2
+orjson==3.11.1
+packaging==25.0
+pandas==2.3.1
+pfzy==0.3.4
+pillow==11.3.0
+prompt-toolkit==3.0.51
+protobuf==6.31.1
+pydantic==2.11.7
+pydantic-core==2.33.2
+pydub==0.25.1
+pygments==2.19.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
+python-multipart==0.0.20
+pytz==2025.2
+pyyaml==6.0.2
+regex==2024.11.6
+requests==2.32.4
+rich==14.1.0
+ruff==0.12.5
+safehttpx==0.1.6
+safetensors==0.5.3
+semantic-version==2.10.0
+sentencepiece==0.2.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+starlette==0.47.2
+tiktoken==0.9.0
+tokenizers==0.21.2
+tomlkit==0.13.3
+tqdm==4.67.1
+transformers==4.54.0
+typer==0.16.0
+typing-extensions==4.14.1
+typing-inspection==0.4.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+wcwidth==0.2.13
+websockets==15.0.1