app.py

import streamlit as st
from tokenizers.tools import EncodingVisualizer
from transformers import AutoTokenizer

@st.cache(allow_output_mutation=True)
def load_tokenizer(model_ckpt):
    return AutoTokenizer.from_pretrained(model_ckpt, use_auth_token=st.secrets["AUTH_TOKEN"])

st.set_page_config(page_title="BigScience Tokenizer", page_icon='👩‍💻', layout="wide")
tokenizer = load_tokenizer('bigscience/tokenizer')
visualizer = EncodingVisualizer(tokenizer=tokenizer._tokenizer, default_to_notebook=False)
text = st.text_area(label="", placeholder="Text to tokenize")
button_clicked = st.button("Tokenize")
if text or button_clicked:
    st.write(len(tokenizer.tokenize(text)))
    st.components.v1.html(visualizer(text), height=1500)