import streamlit as st from transformers import VisionEncoderDecoderModel, AutoTokenizer from datasets import load_dataset, concatenate_datasets from texteller.api.load import load_model, load_tokenizer from texteller.api.inference import img2latex from skimage.metrics import structural_similarity as ssim from modules.cdm.evaluation import compute_cdm_score from PIL import Image import numpy as np import matplotlib.pyplot as plt import seaborn as sns import io from io import BytesIO import base64 import pandas as pd import re import os import evaluate import time from collections import defaultdict import shutil # Configure Streamlit layout st.set_page_config(layout="wide") st.title("TeXTeller Demo: LaTeX Code Prediction from Math Images") # Load model and tokenizer @st.cache_resource def load_model_and_tokenizer(): checkpoint = "OleehyO/TexTeller" model = load_model(checkpoint) tokenizer = load_tokenizer(checkpoint) return model, tokenizer @st.cache_data def load_data(): dataset = load_dataset("linxy/LaTeX_OCR", "small") dataset = concatenate_datasets([split for split in dataset.values()]) dataset = dataset.map(lambda sample: { "complexity": estimate_complexity(sample["text"]), "latex_length": len(sample["text"]), "latex_depth": max_brace_depth(sample["text"]), "text": normalize_latex(sample["text"]) }) return dataset @st.cache_resource def load_metrics(): return evaluate.load("bleu") # Utilities to evaluate LaTeX complexity def count_occurrences(pattern, text): return len(re.findall(pattern, text)) def max_brace_depth(latex): depth = max_depth = 0 for char in latex: if char == '{': depth += 1 max_depth = max(max_depth, depth) elif char == '}': depth -= 1 return max_depth def estimate_complexity(latex): length = len(latex) depth = max_brace_depth(latex) score = 0 score += count_occurrences(r'\\(frac|sqrt)', latex) score += count_occurrences(r'\\(sum|prod|int)', latex) * 2 score += count_occurrences(r'\\(left|right|begin|end)', latex) * 2 score += count_occurrences(r'\\begin\{(bmatrix|matrix|pmatrix)\}', latex) * 3 greek_letters = r'\\(alpha|beta|gamma|delta|epsilon|zeta|eta|theta|iota|kappa|lambda|mu|nu|xi|pi|rho|sigma|tau|upsilon|phi|chi|psi|omega|' \ r'Gamma|Delta|Theta|Lambda|Xi|Pi|Sigma|Upsilon|Phi|Psi|Omega)' score += count_occurrences(greek_letters, latex) * 0.5 score += depth score += length / 20 if score < 4: return "very simple" elif score < 8: return "simple" elif score < 12: return "medium" elif score < 20: return "complex" return "very complex" def normalize_latex(latex_code): latex_code = latex_code.replace(" ", "").replace("\\displaystyle", "") latex_code = re.sub(r"\\begin\{align\**\}", "", latex_code) latex_code = re.sub(r"\\end\{align\**\}", "", latex_code) return latex_code def compute_ssim(image1, image2): """Calcule le SSIM entre deux images PIL""" img1 = np.array(image1.convert("L")) # Convertir en niveaux de gris img2 = np.array(image2.convert("L")) return ssim(img1, img2) # Convert LaTeX to image def latex2image(latex_expression, image_size_in=(3, 0.5), fontsize=16, dpi=200): fig = plt.figure(figsize=image_size_in, dpi=dpi) fig.text( x=0.5, y=0.5, s=f"${latex_expression}$", horizontalalignment="center", verticalalignment="center", fontsize=fontsize ) buf = io.BytesIO() plt.savefig(buf, format="PNG", bbox_inches="tight", pad_inches=0.1) plt.close(fig) buf.seek(0) return Image.open(buf) # --- Convert PIL image to base64 --- def image_to_base64(pil_img: Image.Image) -> str: img = pil_img.copy() with BytesIO() as buffer: img.save(buffer, 'png') return base64.b64encode(buffer.getvalue()).decode() # --- Formatter for HTML rendering --- def image_formatter(pil_img: Image.Image) -> str: img_b64 = image_to_base64(pil_img) return f'' # --- Build HTML table from dictionary --- def build_html_table(metrics_dico): metrics_df = pd.DataFrame(metrics_dico) return metrics_df.to_html(escape=False, formatters={"CDM Image": image_formatter}) model, tokenizer = load_model_and_tokenizer() dataset = load_data() bleu_metric = load_metrics() # Section 1: Dataset Overview st.markdown("---") st.markdown("## 📚 Dataset Overview") st.markdown(""" This demo uses the [LaTeX_OCR dataset](https://huggingface.co/datasets/linxy/LaTeX_OCR) from Hugging Face 🤗. Below are 10 examples showing input images and their corresponding LaTeX code. """) # Take 10 examples sample_dataset = dataset.select(range(10)) # Constrain the width of the "table" to ~50% using centered columns col_left, col_center, col_right = st.columns([1, 2, 1]) with col_center: header1, header2 = st.columns(2, border=True) with header1: st.markdown("

Image

", unsafe_allow_html=True) with header2: st.markdown("

LaTeX Code

", unsafe_allow_html=True) for i in range(10): col1, col2 = st.columns(2, border=True) sample = sample_dataset[i] with col1: st.image(sample["image"]) with col2: st.markdown(f"`{sample['text']}`") # ---- Section 2: Exploratory Data Analysis ---- st.markdown("---") st.header("📊 Exploratory Data Analysis") st.markdown("We analyze the distribution of LaTeX expressions in terms of complexity, length, and depth.") df = pd.DataFrame(dataset) sns.set_theme() # Layout: 3 plots in a row col1, col2, col3 = st.columns(3) with col1: fig, ax = plt.subplots(figsize=(3, 3)) plot = sns.countplot(data=df, x="complexity", order=["very simple", "simple", "medium", "complex", "very complex"], palette="flare", ax=ax) plot.set_xticklabels(plot.get_xticklabels(), rotation=45, horizontalalignment='right', fontsize=8) ax.set_title("LaTeX Formula Complexity", fontsize=8) ax.set_xlabel("") ax.set_ylabel("Count", fontsize=8) st.pyplot(fig) with col2: fig, ax = plt.subplots(figsize=(3, 3)) sns.histplot(df["latex_length"], bins=20, kde=True, ax=ax) ax.set_title("Length of LaTeX Code", fontsize=8) ax.set_xlabel("Characters", fontsize=8) ax.set_ylabel("Count", fontsize=8) st.pyplot(fig) with col3: fig, ax = plt.subplots(figsize=(3, 3)) sns.histplot(df["latex_depth"], bins=5, kde=True, color="forestgreen", ax=ax) ax.set_title("Max Brace Depth of LaTeX Code", fontsize=8) ax.set_xlabel("Depth", fontsize=8) ax.set_ylabel("Count", fontsize=8) st.pyplot(fig) # ---- Section 3: Prediction ---- st.markdown("---") st.header("🔍 TeXTeller Inference") st.markdown("Upload a math image below to predict the LaTeX code using the TeXTeller model.") # Radio button to select input source input_option = st.radio( "Choose an input method:", options=["Upload your own image", "Use a sample from the dataset"], horizontal=True ) image = None selected_index = None if input_option == "Use a sample from the dataset": selected_index = None nb_cols = 5 for i in range(10): # Affiche 10 images if i % nb_cols == 0: cols = st.columns(nb_cols, border=True) col = cols[i % nb_cols] with col: if st.button("Select this sample", key=f"btn_{i}"): selected_index = i st.image(dataset[i]["image"], use_container_width=True) if selected_index is not None: image = dataset[selected_index]["image"] elif input_option == "Upload your own image": uploaded_file = st.file_uploader("Upload a math image (JPG, PNG)...", type=["jpg", "jpeg", "png"]) if uploaded_file: image = Image.open(uploaded_file) image = image.convert("RGB") # Once we have a valid image if image: st.divider() st.markdown("### TeXTeller Prediction Output") col1, col2, col3 = st.columns(3, border=True) with col1: st.image(image, caption="Input Image", use_container_width=True) with st.spinner("Running TeXTeller..."): try: dico_result = defaultdict(list) start = time.time() predicted_latex = img2latex(model, tokenizer, [np.array(image)], out_format="katex")[0] eval_time = time.time() - start dico_result["Inference Time (s)"].append(f"{eval_time:.2f}") with col2: st.markdown("**Predicted LaTeX Code:**") st.text_area(label="", value=predicted_latex, height=80) with col3: rendered_image = latex2image(predicted_latex) st.image(rendered_image, caption="Rendered from Prediction", use_container_width=True) if selected_index is not None: ref_latex = dataset[selected_index]["text"] predicted_latex = normalize_latex(predicted_latex) # Compute BLEU score bleu_results = bleu_metric.compute(predictions=[predicted_latex], references=[[ref_latex]]) bleu_score = bleu_results['bleu'] dico_result["BLEU Score"].append(bleu_score) # Compute SSIM pred_image = rendered_image.resize(image.size) ssim_score = compute_ssim(image, pred_image) dico_result["SSIM Score"].append(ssim_score) # Compute CDM cdm_score, cdm_recall, cdm_precision, compare_img = compute_cdm_score(ref_latex, predicted_latex) dico_result["CDM Image"].append(compare_img) dico_result["CDM Score"].append(cdm_score) # Display metrics html = build_html_table(dico_result) st.markdown("### TeXTeller Metrics") # CSS pour forcer le tableau à occuper toute la largeur st.markdown(""" """, unsafe_allow_html=True) st.markdown(html, unsafe_allow_html=True) except Exception as e: st.error(f"Error during prediction: {e}")