Create app.py

app.py

@@ -0,0 +1,53 @@
+import gradio as gr
+from transformers import AutoTokenizer
+
+tokenizers = {
+    "GPT-2 Tokenizer": AutoTokenizer.from_pretrained("gpt2"),
+    "RoBERTa Tokenizer": AutoTokenizer.from_pretrained("roberta-base"),
+    "DistilGPT-2 Tokenizer": AutoTokenizer.from_pretrained("distilgpt2"),
+    "bert-base-german-cased Tokenizer": AutoTokenizer.from_pretrained("bert-base-german-cased")
+    "GPT-4 Tokenizer":GPT2TokenizerFast.from_pretrained('Xenova/gpt-4')
+}
+
+# Fancy token visualization function
+def visualize_tokens(text, tokenizer_name, show_token_ids):
+    tokenizer = tokenizers[tokenizer_name]
+    encoded = tokenizer(text, add_special_tokens=False, return_tensors=None)
+    token_ids = encoded["input_ids"]
+    tokens = tokenizer.convert_ids_to_tokens(token_ids)
+
+    # Display each token inside a colored span
+    html_tokens = []
+    for idx, token in enumerate(tokens):
+        html_token = f"<span style='display:inline-block; margin:2px; padding:4px; background-color:#eee; border-radius:6px;'>{token}</span>"
+        html_tokens.append(html_token)
+
+    html_output = " ".join(html_tokens)
+    
+    if show_token_ids:
+        html_output += "<br><br><b>Token IDs:</b><br>" + str(token_ids)
+
+    return html_output, f"🔢 Token Count: {len(tokens)}"
+
+# Gradio app
+with gr.Blocks() as app:
+    gr.Markdown("# 🚀 Tokenizer Playground (Tiktokenizer-Style)")
+    
+    with gr.Row():
+        with gr.Column():
+            text_input = gr.Textbox(lines=4, label="Enter your text here", placeholder="Type or paste text...")
+            tokenizer_choice = gr.Dropdown(list(tokenizers.keys()), label="Choose Tokenizer")
+            show_ids = gr.Checkbox(label="Show Token IDs", value=False)
+            tokenize_btn = gr.Button("Tokenize!")
+        with gr.Column():
+            html_output = gr.HTML(label="Tokens Visualized")
+            token_count = gr.Label(label="Token Count")
+
+    tokenize_btn.click(
+        visualize_tokens,
+        inputs=[text_input, tokenizer_choice, show_ids],
+        outputs=[html_output, token_count]
+    )
+
+# Launch
+app.launch()