File size: 1,236 Bytes
276a3be |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
import gradio as gr
import pandas as pd
from sentence_transformers import SentenceTransformer
# 1. Load a pretrained SentenceTransformer model
model = SentenceTransformer('all-MiniLM-L6-v2')
def clean_and_embed(text: str):
# 2. Clean: remove non-ASCII & lowercase
clean = text.encode('ascii', 'ignore').decode().lower()
# 3. Tokenize via the model’s tokenizer
tokens = model.tokenizer.tokenize(clean)
# 4. Get the sentence embedding
emb = model.encode(clean, convert_to_numpy=True)
# 5. Build a DataFrame: one row (the sentence) × embedding dims
df = pd.DataFrame(
[emb],
index=["sentence_embedding"],
columns=[f"dim_{i}" for i in range(emb.shape[0])]
)
return " ".join(tokens), df
# 6. Gradio interface
iface = gr.Interface(
fn=clean_and_embed,
inputs=gr.Textbox(lines=2, placeholder="Type your text here…"),
outputs=[
gr.Textbox(label="Tokens"),
gr.Dataframe(label="Sentence Embedding Vector")
],
title="ASCII‑Clean + SentenceTransformer",
description="Cleans input, tokenizes with a SentenceTransformer tokenizer, and shows the sentence embedding."
)
if __name__ == "__main__":
iface.launch()
|