Update app.py

app.py

@@ -77,35 +77,152 @@ def validate_file(file_obj) -> None:
     if file_size > MAX_FILE_SIZE_MB:
         raise gr.Error(f"File too large. Max size: {MAX_FILE_SIZE_MB}MB")
 
+# ========== ENHANCED TRANSCRIPT PARSING ==========
+def extract_text_from_file(file_path: str, file_ext: str) -> str:
+    """Enhanced text extraction with better error handling and fallbacks."""
+    text = ""
+    
+    try:
+        if file_ext == '.pdf':
+            # First try PyMuPDF for better text extraction
+            try:
+                doc = fitz.open(file_path)
+                for page in doc:
+                    text += page.get_text("text") + '\n'
+                if not text.strip():
+                    raise ValueError("PyMuPDF returned empty text")
+            except Exception as e:
+                print(f"PyMuPDF failed, trying OCR fallback: {str(e)}")
+                text = extract_text_from_pdf_with_ocr(file_path)
+        
+        elif file_ext in ['.png', '.jpg', '.jpeg']:
+            text = extract_text_with_ocr(file_path)
+            
+        # Clean up the extracted text
+        text = clean_extracted_text(text)
+        
+        if not text.strip():
+            raise ValueError("No text could be extracted from the file")
+            
+        return text
+    
+    except Exception as e:
+        raise gr.Error(f"Text extraction error: {str(e)}")
+
+def extract_text_from_pdf_with_ocr(file_path: str) -> str:
+    """Fallback PDF text extraction using OCR."""
+    text = ""
+    try:
+        doc = fitz.open(file_path)
+        for page in doc:
+            pix = page.get_pixmap()
+            img = Image.open(io.BytesIO(pix.tobytes()))
+            text += pytesseract.image_to_string(img) + '\n'
+    except Exception as e:
+        raise ValueError(f"PDF OCR failed: {str(e)}")
+    return text
+
 def extract_text_with_ocr(file_path: str) -> str:
-    """Extract text from image files using OCR."""
+    """Extract text from image files using OCR with preprocessing."""
     try:
         image = Image.open(file_path)
-        text = pytesseract.image_to_string(image)
+        
+        # Preprocess image for better OCR results
+        image = image.convert('L')  # Convert to grayscale
+        image = image.point(lambda x: 0 if x < 128 else 255, '1')  # Thresholding
+        
+        # Custom Tesseract configuration
+        custom_config = r'--oem 3 --psm 6'
+        text = pytesseract.image_to_string(image, config=custom_config)
         return text
     except Exception as e:
-        raise gr.Error(f"OCR processing failed: {str(e)}")
+        raise ValueError(f"OCR processing failed: {str(e)}")
+
+def clean_extracted_text(text: str) -> str:
+    """Clean and normalize the extracted text."""
+    # Remove multiple spaces and newlines
+    text = re.sub(r'\s+', ' ', text).strip()
+    
+    # Fix common OCR errors
+    replacements = {
+        '|': 'I',
+        '‘': "'",
+        '’': "'",
+        '“': '"',
+        '”': '"',
+        'fi': 'fi',
+        'fl': 'fl'
+    }
+    
+    for wrong, right in replacements.items():
+        text = text.replace(wrong, right)
+    
+    return text
+
+def remove_sensitive_info(text: str) -> str:
+    """Remove potentially sensitive information from transcript text."""
+    # Remove social security numbers
+    text = re.sub(r'\b\d{3}-\d{2}-\d{4}\b', '[REDACTED]', text)
+    # Remove student IDs (assuming 6-9 digit numbers)
+    text = re.sub(r'\b\d{6,9}\b', '[ID]', text)
+    # Remove email addresses
+    text = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '[EMAIL]', text)
+    return text
+
+def extract_json_from_response(content: str) -> str:
+    """Extract JSON string from API response."""
+    # Handle markdown code blocks
+    if '```json' in content:
+        content = content.split('```json')[1].split('```')[0].strip()
+    elif '```' in content:
+        content = content.split('```')[1].split('```')[0].strip()
+    
+    # Sometimes the response is pure JSON
+    return content
+
+def validate_parsed_data(data: Dict) -> Dict:
+    """Validate and clean the parsed data structure."""
+    # Ensure required fields exist
+    if not isinstance(data, dict):
+        raise ValueError("Invalid data format")
+    
+    # Set default structure if missing
+    if 'grade_level' not in data:
+        data['grade_level'] = 'Unknown'
+    
+    if 'gpa' not in data:
+        data['gpa'] = {'weighted': 'N/A', 'unweighted': 'N/A'}
+    
+    if 'courses' not in data:
+        data['courses'] = []
+    
+    # Clean course data
+    for course in data['courses']:
+        if 'grade' in course:
+            course['grade'] = course['grade'].upper().strip()
+        
+        # Ensure numeric credits are strings
+        if 'credits' in course and isinstance(course['credits'], (int, float)):
+            course['credits'] = str(course['credits'])
+    
+    return data
 
-# ========== ENHANCED TRANSCRIPT PARSING WITH DEEPSEEK ==========
 def parse_transcript_with_deepseek(text: str) -> Dict:
-    """Use DeepSeek model to parse transcript text with high accuracy."""
+    """Improved DeepSeek API integration with better error handling."""
     if not DEEPSEEK_API_KEY:
         raise gr.Error("DeepSeek API key not configured")
     
+    # Pre-process the text to remove sensitive information
+    text = remove_sensitive_info(text)
+    
+    # Create a more robust prompt with examples
     prompt = f"""
-    Analyze this academic transcript and extract the following information in JSON format:
-    - Current grade level
-    - Weighted GPA
-    - Unweighted GPA
-    - List of all courses with:
-      * Course code
-      * Course name
-      * Grade received
-      * Credits earned
-      * Year/semester taken
-      * Grade level when taken
-
-    Return the data in this exact JSON structure:
+    Analyze this academic transcript and extract structured information. Follow these rules:
+    1. Extract data even if partially visible
+    2. Guess missing values when reasonable
+    3. Return empty if completely missing
+    
+    Required JSON structure:
     {{
         "grade_level": "11",
         "gpa": {{
@@ -120,13 +237,12 @@ def parse_transcript_with_deepseek(text: str) -> Dict:
                 "credits": "1.0",
                 "year": "2023-2024",
                 "grade_level": "11"
-            }},
-            // more courses...
+            }}
         ]
     }}
-
-    Here is the transcript text to analyze:
-    {text}
+    
+    Transcript Text:
+    {text[:15000]}  # Limit to first 15k chars to avoid token limits
     """
     
     headers = {
@@ -142,94 +258,92 @@ def parse_transcript_with_deepseek(text: str) -> Dict:
     }
     
     try:
-        response = requests.post(DEEPSEEK_API_URL, headers=headers, json=payload)
+        response = requests.post(DEEPSEEK_API_URL, headers=headers, json=payload, timeout=30)
         response.raise_for_status()
         result = response.json()
         
-        # Extract the JSON content from the response
         content = result['choices'][0]['message']['content']
         
-        # Sometimes the response includes markdown code blocks
-        if '```json' in content:
-            content = content.split('```json')[1].split('```')[0].strip()
-        elif '```' in content:
-            content = content.split('```')[1].split('```')[0].strip()
+        # Extract JSON from response (handling markdown code blocks)
+        json_str = extract_json_from_response(content)
         
-        return json.loads(content)
+        # Validate and clean the parsed data
+        parsed_data = validate_parsed_data(json.loads(json_str))
+        
+        return parsed_data
+    
+    except requests.exceptions.RequestException as e:
+        raise gr.Error(f"API request failed: {str(e)}")
+    except json.JSONDecodeError as e:
+        raise gr.Error(f"Failed to parse API response: {str(e)}")
     except Exception as e:
-        raise gr.Error(f"DeepSeek API error: {str(e)}")
+        raise gr.Error(f"DeepSeek processing error: {str(e)}")
+
+def format_transcript_output(data: Dict) -> str:
+    """Format the parsed data into human-readable text."""
+    output = []
+    output.append(f"Student Transcript Summary\n{'='*40}")
+    output.append(f"Current Grade Level: {data.get('grade_level', 'Unknown')}")
+    
+    if 'gpa' in data:
+        output.append(f"\nGPA:")
+        output.append(f"- Weighted: {data['gpa'].get('weighted', 'N/A')}")
+        output.append(f"- Unweighted: {data['gpa'].get('unweighted', 'N/A')}")
+    
+    if 'courses' in data:
+        output.append("\nCourse History:\n" + '='*40)
+        
+        # Group courses by grade level
+        courses_by_grade = defaultdict(list)
+        for course in data['courses']:
+            grade_level = course.get('grade_level', 'Unknown')
+            courses_by_grade[grade_level].append(course)
+        
+        # Sort grades numerically
+        for grade in sorted(courses_by_grade.keys(), key=lambda x: int(x) if x.isdigit() else x):
+            output.append(f"\nGrade {grade}:\n{'-'*30}")
+            for course in courses_by_grade[grade]:
+                course_str = f"- {course.get('code', '')} {course.get('name', 'Unnamed course')}"
+                if 'grade' in course:
+                    course_str += f" (Grade: {course['grade']})"
+                if 'credits' in course:
+                    course_str += f" | Credits: {course['credits']}"
+                if 'year' in course:
+                    course_str += f" | Year: {course['year']}"
+                output.append(course_str)
+    
+    return '\n'.join(output)
 
 def parse_transcript(file_obj) -> Tuple[str, Optional[Dict]]:
-    """Parse transcript file with DeepSeek enhanced parsing."""
+    """Main function to parse transcript files."""
     try:
         if not file_obj:
-            raise gr.Error("Please upload a file first")
+            raise ValueError("Please upload a file first")
             
         validate_file(file_obj)
-        
-        text = ''
         file_ext = os.path.splitext(file_obj.name)[1].lower()
         
-        try:
-            if file_ext == '.pdf':
-                # Try PyMuPDF first for better text extraction
-                try:
-                    doc = fitz.open(file_obj.name)
-                    for page in doc:
-                        text += page.get_text() + '\n'
-                except:
-                    # Fallback to PyPDF2
-                    reader = PdfReader(file_obj.name)
-                    for page in reader.pages:
-                        page_text = page.extract_text()
-                        if page_text:
-                            text += page_text + '\n'
-            elif file_ext in ['.png', '.jpg', '.jpeg']:
-                text = extract_text_with_ocr(file_obj.name)
-        except Exception as e:
-            raise gr.Error(f"Error processing file: {str(e)}")
+        # Extract text from file
+        text = extract_text_from_file(file_obj.name, file_ext)
         
-        if not text.strip():
-            raise gr.Error("No text could be extracted from the file")
-        
-        # Use DeepSeek for enhanced parsing
+        # Parse with DeepSeek
         parsed_data = parse_transcript_with_deepseek(text)
         
-        # Format output text
-        output_text = f"Student Transcript Summary\n{'='*40}\n"
-        output_text += f"Current Grade Level: {parsed_data.get('grade_level', 'Unknown')}\n"
-        
-        if 'gpa' in parsed_data:
-            output_text += f"Weighted GPA: {parsed_data['gpa'].get('weighted', 'N/A')}\n"
-            output_text += f"Unweighted GPA: {parsed_data['gpa'].get('unweighted', 'N/A')}\n\n"
-        
-        output_text += "Course History:\n{'='*40}\n"
-        
-        # Organize courses by grade level
-        courses_by_grade = defaultdict(list)
-        for course in parsed_data.get('courses', []):
-            grade_level = course.get('grade_level', 'Unknown')
-            courses_by_grade[grade_level].append(course)
-        
-        for grade in sorted(courses_by_grade.keys(), key=lambda x: int(x) if x.isdigit() else x):
-            output_text += f"\nGrade {grade}:\n{'-'*30}\n"
-            for course in courses_by_grade[grade]:
-                output_text += f"- {course.get('code', '')} {course.get('name', 'Unnamed course')}"
-                if 'grade' in course and course['grade']:
-                    output_text += f" (Grade: {course['grade']})"
-                if 'credits' in course:
-                    output_text += f" | Credits: {course['credits']}"
-                if 'year' in course:
-                    output_text += f" | Year: {course['year']}"
-                output_text += "\n"
+        # Format output
+        output_text = format_transcript_output(parsed_data)
         
         # Prepare the data structure for saving
         transcript_data = {
             "grade_level": parsed_data.get('grade_level', 'Unknown'),
             "gpa": parsed_data.get('gpa', {}),
-            "courses": dict(courses_by_grade)
+            "courses": defaultdict(list)
         }
         
+        # Organize courses by grade level for saving
+        for course in parsed_data.get('courses', []):
+            grade_level = course.get('grade_level', 'Unknown')
+            transcript_data["courses"][grade_level].append(course)
+        
         return output_text, transcript_data
     
     except Exception as e: