"""from fastapi import FastAPI from fastapi.responses import RedirectResponse import gradio as gr from transformers import pipeline, ViltProcessor, ViltForQuestionAnswering, AutoTokenizer, AutoModelForCausalLM from PIL import Image import torch import fitz # PyMuPDF for PDF app = FastAPI() # ========== Image QA Setup ========== vqa_processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") vqa_model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa") def answer_question_from_image(image, question): if image is None or not question.strip(): return "Please upload an image and ask a question." inputs = vqa_processor(image, question, return_tensors="pt") with torch.no_grad(): outputs = vqa_model(**inputs) predicted_id = outputs.logits.argmax(-1).item() return vqa_model.config.id2label[predicted_id] # ========== Gradio Interfaces ========== img_interface = gr.Interface( fn=answer_question_from_image, inputs=[gr.Image(label="Upload Image"), gr.Textbox(label="Ask a Question")], outputs="text", title="Image Question Answering" ) # ========== Combine and Mount ========== demo = gr.TabbedInterface( img_interface , "Image QA") app = gr.mount_gradio_app(app, demo, path="/") @app.get("/") def root(): return RedirectResponse(url="/") """ """from transformers import ViltProcessor, ViltForQuestionAnswering import torch # Load image QA model once vqa_processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") vqa_model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa") def answer_question_from_image(image, question): if image is None or not question.strip(): return "Please upload an image and ask a question." inputs = vqa_processor(image, question, return_tensors="pt") with torch.no_grad(): outputs = vqa_model(**inputs) predicted_id = outputs.logits.argmax(-1).item() return vqa_model.config.id2label[predicted_id]""" ### ✅ appImage.py — Image QA Backend (Cleaned) from fastapi import FastAPI from fastapi.responses import RedirectResponse, JSONResponse, FileResponse import os from PIL import Image from transformers import ViltProcessor, ViltForQuestionAnswering, pipeline from gtts import gTTS import easyocr import torch import tempfile import numpy as np from io import BytesIO app = FastAPI() vqa_processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-finetuned-vqa") vqa_model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa") captioner = pipeline("image-to-text", model="nlpconnect/vit-gpt2-image-captioning") reader = easyocr.Reader(['en', 'fr']) def classify_question(question: str): q = question.lower() if any(w in q for w in ["text", "say", "written", "read"]): return "ocr" if any(w in q for w in ["caption", "describe", "what is in the image"]): return "caption" return "vqa" def answer_question_from_image(image, question): if image is None or not question.strip(): return "Please upload an image and ask a question.", None mode = classify_question(question) try: if mode == "ocr": result = reader.readtext(np.array(image)) answer = " ".join([entry[1] for entry in result]) or "No readable text found." elif mode == "caption": answer = captioner(image)[0]['generated_text'] else: inputs = vqa_processor(image, question, return_tensors="pt") with torch.no_grad(): outputs = vqa_model(**inputs) predicted_id = outputs.logits.argmax(-1).item() answer = vqa_model.config.id2label[predicted_id] tts = gTTS(text=answer) with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp: tts.save(tmp.name) return answer, tmp.name except Exception as e: return f"Error: {e}", None @app.get("/") def home(): return RedirectResponse(url="/templates/home.html")