Fix: Replaced NLTK tokenizer with Hugging Face AutoTokenizer

app.py

@@ -4,23 +4,12 @@ import pdfplumber
 from flask import Flask, render_template, request, redirect, url_for, flash, send_file
 from werkzeug.utils import secure_filename
 from sentence_transformers import SentenceTransformer, util
-import nltk
-from nltk.stem import WordNetLemmatizer, PorterStemmer
-from nltk.tokenize import word_tokenize
-from nltk.corpus import stopwords
+from transformers import AutoTokenizer
 from fpdf import FPDF
 from collections import Counter
 
-# Download risorse NLTK (incluso punkt)
-nltk.download('punkt', download_dir='/home/user/nltk_data')
-nltk.download('wordnet', download_dir='/home/user/nltk_data')
-nltk.download('stopwords', download_dir='/home/user/nltk_data')
-
-nltk.data.path.append("/home/user/nltk_data")
-
-lemmatizer = WordNetLemmatizer()
-stemmer = PorterStemmer()
-stop_words = set(stopwords.words("english"))
+# Usa Hugging Face tokenizer
+tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
 
 app = Flask(__name__)
 app.secret_key = os.environ.get("SECRET_KEY", "NORUS_secretkey_05")
@@ -43,9 +32,9 @@ def extract_pdf_text(pdf_path):
     return text.lower().strip()
 
 def preprocess_text(text):
-    words = word_tokenize(text.lower())
-    words = [stemmer.stem(lemmatizer.lemmatize(w)) for w in words if w.isalnum() and w not in stop_words and len(w) > 3]
-    return words
+    # Tokenizza il testo usando il tokenizer di Hugging Face
+    tokens = tokenizer.tokenize(text.lower())
+    return [token for token in tokens if len(token) > 3]
 
 def calculate_token_overlap(text1, text2):
     tokens1 = set(text1.split())