changed the grammar checking model

app.py

@@ -3,7 +3,8 @@ import torch
 from transformers import (
     AutoTokenizer, 
     AutoModelForSequenceClassification,
-    pipeline
+    T5ForConditionalGeneration,
+    T5Tokenizer
 )
 import os
 from pdf_generator import ReportGenerator
@@ -18,13 +19,13 @@ def load_models():
     hate_tokenizer = AutoTokenizer.from_pretrained("facebook/roberta-hate-speech-dynabench-r4-target")
     hate_model = AutoModelForSequenceClassification.from_pretrained("facebook/roberta-hate-speech-dynabench-r4-target")
     
-    # Bias detection (using same model with different labels)
-    bias_tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
-    bias_model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased")
+    # Grammar check model (using T5)
+    grammar_tokenizer = T5Tokenizer.from_pretrained("orthwand/t5-small-grammar-correction")
+    grammar_model = T5ForConditionalGeneration.from_pretrained("orthwand/t5-small-grammar-correction")
     
     return {
         'hate_speech': (hate_model, hate_tokenizer),
-        'bias': (bias_model, bias_tokenizer)
+        'grammar': (grammar_model, grammar_tokenizer)
     }
 
 # Initialize news checker
@@ -68,12 +69,20 @@ def check_hate_speech(text, model, tokenizer):
             'message': f'Error in hate speech detection: {str(e)}'
         }
 
-def check_grammar(text):
+def check_grammar(text, model, tokenizer):
     try:
-        nlp = pipeline("text2text-generation", model="gramformer/gramformer", device=0 if torch.cuda.is_available() else -1)
-        corrected = nlp(text, max_length=1000)[0]['generated_text']
+        input_ids = tokenizer(f"grammar: {text}", return_tensors="pt", max_length=512, truncation=True).input_ids
         
-        if corrected.lower() != text.lower():
+        outputs = model.generate(
+            input_ids,
+            max_length=512,
+            num_beams=4,
+            early_stopping=True
+        )
+        
+        corrected = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        
+        if corrected.lower().strip() != text.lower().strip():
             return {
                 'status': 'warning',
                 'message': f'Suggested corrections:\n{corrected}'
@@ -115,12 +124,18 @@ def analyze_content(text):
     report_gen.add_check_result("Hate Speech Check", hate_result['status'], hate_result['message'])
     
     # 3. Grammar Check
-    grammar_result = check_grammar(text)
+    grammar_result = check_grammar(text, models['grammar'][0], models['grammar'][1])
     results['Grammar Check'] = grammar_result
     report_gen.add_check_result("Grammar Check", grammar_result['status'], grammar_result['message'])
     
     # 4. News Context Check
-    news_result = news_checker.check_content_against_news(text)
+    if os.getenv('NEWS_API_KEY'):
+        news_result = news_checker.check_content_against_news(text)
+    else:
+        news_result = {
+            'status': 'warning',
+            'message': 'News API key not configured. Skipping current events check.'
+        }
     results['Current Events Context'] = news_result
     report_gen.add_check_result("Current Events Context", news_result['status'], news_result['message'])
     
@@ -186,7 +201,7 @@ def create_interface():
           - Text length
           - Hate speech and bias
           - Grammar
-          - Current events context
+          - Current events context (requires News API key)
         """)
     
     return interface

news_checker.py

@@ -9,9 +9,21 @@ load_dotenv()
 class NewsChecker:
     def __init__(self):
         self.api_key = os.getenv('NEWS_API_KEY')
-        self.newsapi = NewsApiClient(api_key=self.api_key)
+        if not self.api_key:
+            print("WARNING: NEWS_API_KEY not found in environment variables")
+        else:
+            print("NEWS_API_KEY found in environment variables")
+            
+        try:
+            self.newsapi = NewsApiClient(api_key=self.api_key)
+        except Exception as e:
+            print(f"Error initializing NewsAPI client: {str(e)}")
         
     def get_recent_news(self):
+        if not self.api_key:
+            print("Cannot fetch news: No API key configured")
+            return pd.DataFrame()
+            
         try:
             # Get news from the last 7 days
             week_ago = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
@@ -33,9 +45,12 @@ class NewsChecker:
                     }
                     for article in articles if article['description']
                 ]
+                print(f"Successfully fetched {len(news_data)} articles")
                 return pd.DataFrame(news_data)
-            return pd.DataFrame()
-            
+            else:
+                print(f"NewsAPI response status was not 'ok': {response.get('status')}")
+                return pd.DataFrame()
+                
         except Exception as e:
             print(f"Error fetching news: {str(e)}")
             return pd.DataFrame()
@@ -49,7 +64,6 @@ class NewsChecker:
             }
             
         # Simple keyword matching for demo purposes
-        # In a production environment, you'd want to use more sophisticated NLP techniques
         marketing_words = set(marketing_text.lower().split())
         potential_conflicts = []
         

requirements.txt

@@ -6,4 +6,6 @@ fpdf2==2.7.8
 pandas==2.1.4
 numpy==1.24.3
 requests==2.31.0
-python-dotenv==1.0.0
+python-dotenv==1.0.0
+sentencepiece==0.2.0
+sacremoses==0.1.1