Upload 3 files

models/image_analyzer.py

@@ -0,0 +1,43 @@
+import torch
+from transformers import AutoFeatureExtractor, AutoModelForImageClassification
+
+
+class ImageAnalyzer:
+    def __init__(self):
+        # Load the chest X-ray analysis model
+        try:
+            model_name = "facebook/deit-base-patch16-224-medical-cxr"
+            self.feature_extractor = AutoFeatureExtractor.from_pretrained(model_name)
+            self.model = AutoModelForImageClassification.from_pretrained(model_name)
+            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            self.model.to(self.device)
+            print(f"Image model loaded on {self.device}")
+        except Exception as e:
+            print(f"Error loading image model: {e}")
+            # Fallback to a dummy model
+            self.model = None
+            self.feature_extractor = None
+
+    def analyze(self, image):
+        """Analyze an X-ray image and return predictions with confidence scores"""
+        if self.model is None or self.feature_extractor is None:
+            return {"No findings": 0.7, "Abnormal": 0.3}  # Dummy results
+
+        try:
+            inputs = self.feature_extractor(images=image, return_tensors="pt").to(
+                self.device
+            )
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+
+            # Process outputs to get predicted class and confidence
+            probabilities = torch.nn.functional.softmax(outputs.logits, dim=1)[0]
+            predictions = {}
+            for i, prob in enumerate(probabilities):
+                label = self.model.config.id2label[i]
+                predictions[label] = float(prob)
+
+            return predictions
+        except Exception as e:
+            print(f"Error during image analysis: {e}")
+            return {"Error": "Could not analyze image"}

models/multimodal_fusion.py

@@ -0,0 +1,58 @@
+class MultimodalFusion:
+    """
+    Combines insights from image analysis and text analysis
+    to provide comprehensive medical assessment
+    """
+
+    def __init__(self):
+        pass
+
+    def fuse_insights(self, image_results, text_results):
+        """
+        Fuse insights from image and text analysis
+
+        Args:
+            image_results (dict): Results from image analysis
+            text_results (dict): Results from text analysis
+
+        Returns:
+            dict: Combined insights with recommendation
+        """
+        # In a real implementation, this would use more sophisticated fusion techniques
+        combined_insights = {
+            "Image findings": image_results,
+            "Text findings": text_results,
+        }
+
+        # Simple fusion logic
+        confidence_scores = [
+            value for key, value in image_results.items() if isinstance(value, float)
+        ]
+        avg_confidence = (
+            sum(confidence_scores) / len(confidence_scores) if confidence_scores else 0
+        )
+
+        # Determine if any abnormalities are detected in image
+        image_abnormal = any(
+            key != "No findings" and value > 0.5
+            for key, value in image_results.items()
+            if isinstance(value, float)
+        )
+
+        # Check if text analysis found concerning elements
+        text_concerning = text_results.get("Sentiment") == "Concerning"
+
+        # Generate recommendation
+        if image_abnormal and text_concerning:
+            recommendation = "High priority: Both image and text indicate abnormalities"
+        elif image_abnormal:
+            recommendation = "Medium priority: Image shows potential abnormalities"
+        elif text_concerning:
+            recommendation = "Medium priority: Text report indicates concerns"
+        else:
+            recommendation = "Low priority: No significant findings detected"
+
+        combined_insights["Recommendation"] = recommendation
+        combined_insights["Confidence"] = f"{avg_confidence:.2f}"
+
+        return combined_insights

models/text_analyzer.py

@@ -0,0 +1,72 @@
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
+
+
+class TextAnalyzer:
+    def __init__(self):
+        # Load the medical text analysis model
+        try:
+            model_name = "medicalai/ClinicalBERT"
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+            self.model = AutoModelForSequenceClassification.from_pretrained(model_name)
+            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            self.model.to(self.device)
+
+            # NER pipeline for medical entities
+            self.ner_pipeline = pipeline(
+                "ner", model="samrawal/bert-base-uncased_medical-ner"
+            )
+            print(f"Text model loaded on {self.device}")
+        except Exception as e:
+            print(f"Error loading text model: {e}")
+            # Fallback to dummy functionality
+            self.model = None
+            self.tokenizer = None
+            self.ner_pipeline = None
+
+    def analyze(self, text):
+        """Analyze medical report text and extract key insights"""
+        if text.strip() == "":
+            return {"Insights": "No text provided"}
+
+        if self.model is None or self.tokenizer is None:
+            # Dummy analysis
+            return {
+                "Entities": ["fever", "cough"],
+                "Sentiment": "Concerning",
+                "Key findings": "Patient shows symptoms of respiratory illness",
+            }
+
+        try:
+            # Extract medical entities
+            if self.ner_pipeline:
+                entities = self.ner_pipeline(text)
+                unique_entities = list(set([entity["word"] for entity in entities]))
+            else:
+                unique_entities = []
+
+            # Simple text classification (in real app, would be more sophisticated)
+            inputs = self.tokenizer(
+                text, return_tensors="pt", padding=True, truncation=True
+            ).to(self.device)
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+
+            # This is a placeholder - in reality would depend on the actual model output
+            sentiment = (
+                "Concerning" if torch.sigmoid(outputs.logits).item() > 0.5 else "Normal"
+            )
+
+            # Generate key findings (simplified)
+            key_findings = f"Report indicates {'abnormal' if sentiment == 'Concerning' else 'normal'} findings"
+            if unique_entities:
+                key_findings += f" with mentions of {', '.join(unique_entities[:5])}"
+
+            return {
+                "Entities": unique_entities[:10],
+                "Sentiment": sentiment,
+                "Key findings": key_findings,
+            }
+        except Exception as e:
+            print(f"Error during text analysis: {e}")
+            return {"Error": "Could not analyze text"}