Update app.py

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, GPT2TokenizerFast
 import random
 
 # List of available tokenizers
@@ -54,7 +54,10 @@ def generate_colored_html(tokens, decoded_tokens):
     return html_output
 
 def tokenize_text(text, tokenizer_name):
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+    if tokenizer_name.split("/")[0]=="Xenova":
+        tokenizer = GPT2TokenizerFast.from_pretrained(tokenizer_name)
+    else :
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
     tokens = tokenizer.encode(text, add_special_tokens=True)
     decoded_tokens = [tokenizer.decode(token) for token in tokens]
     html_output = generate_colored_html(tokens, decoded_tokens)