run inference using transformers and not unsloth

app.py

@@ -1,18 +1,16 @@
 import gradio as gr
 import spaces
-from unsloth import FastVisionModel
-import torch
+from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
+from qwen_vl_utils import process_vision_info
 from PIL import Image
 import numpy as np
 import traceback
 from typing import Any, Optional
 
-# Import user-provided modules
 import utils
-from utils import Finding, BoundingBox  # Explicitly import needed classes
+from utils import BoundingBox
 import blurnonymize
 
-# --- Constants ---
 MODEL_NAME = "cborg/qwen2.5VL-3b-privacydetector"
 MAX_NEW_TOKENS = 2048
 TEMPERATURE = 1.0
@@ -80,11 +78,12 @@ def build_messages(image, history: Optional[list[dict[str, Any]]] = None, prompt
 # --- Model Loading ---
 # Load model using unsloth for 4-bit quantization
 try:
-    model, tokenizer = FastVisionModel.from_pretrained(
-        model_name=MODEL_NAME,
-        load_in_4bit=True,
+    # default: Load the model on the available device(s)
+    model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+        MODEL_NAME, torch_dtype="auto", device_map="auto"
     )
-    FastVisionModel.for_inference(model)
+    tokenizer = AutoProcessor.from_pretrained(MODEL_NAME)
+
     model.to("cuda").eval()  # Ensure model is on GPU and in eval mode
     print("Model loaded successfully.")
 except Exception as e:
@@ -124,12 +123,16 @@ def run_model_inference(input_image_pil: Image.Image, prompt_text: str):
     messages = build_messages(
         input_image_pil,
         prompt=prompt_text)
-    input_text = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
 
-    # Prepare inputs for the model
+    input_text = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+    image_inputs, video_inputs = process_vision_info(messages)
     inputs = tokenizer(
-        input_image_pil,
-        input_text,
+        text=[input_text],
+        images=image_inputs,
+        videos=video_inputs,
+        padding=True,
         return_tensors="pt",
     ).to("cuda")
 

requirements.txt

@@ -1,5 +1,4 @@
 gradio
-unsloth
 transformers
 torch
 pydantic
@@ -10,3 +9,4 @@ opencv-python
 spaces
 matplotlib
 sam2
+qwen-vl-utils[decord]==0.0.8