File size: 824 Bytes
4c95418 1f2d2e0 4c95418 583e12e 4c95418 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
from transformers import AutoModelForSequenceClassification, AutoTokenizer
def load_model_and_tokenizer():
"""
Load the fine-tuned XLM-RoBERTa model and tokenizer.
Returns the model and tokenizer for use in classification.
"""
try:
model_name = "JanviMl/xlm-roberta-toxic-classifier-capstone" # Replace with your model repo ID
# If the model is local: model_name = "./model"
model = AutoModelForSequenceClassification.from_pretrained(model_name)
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) # Use slow tokenizer
return model, tokenizer
except Exception as e:
raise Exception(f"Error loading model or tokenizer: {str(e)}")
# Load the model and tokenizer once at startup
model, tokenizer = load_model_and_tokenizer() |