Update app.py

app.py

@@ -1,35 +1,17 @@
 import streamlit as st
 import torch
-from transformers import pipeline, AutoProcessor, AutoModel
+from transformers import pipeline
 from PIL import Image
 import numpy as np
 import cv2
 import tempfile
 import os
-
-def save_video_frames(frames, output_path, fps=30):
-    """
-    Save video frames using OpenCV instead of moviepy
-    """
-    # Get frame dimensions
-    height, width = frames[0].shape[:2]
-    
-    # Initialize video writer
-    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
-    out = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
-    
-    # Write frames
-    for frame in frames:
-        # Convert from RGB to BGR (OpenCV uses BGR)
-        frame_bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
-        out.write(frame_bgr)
-    
-    # Release video writer
-    out.release()
+from diffusers import VideoToVideoSDPipeline
+from diffusers.utils import export_to_video
 
 def generate_video_from_image(image, duration_seconds=10, progress_bar=None):
     """
-    Generate a video from an image using LTX-Video and image captioning.
+    Generate a video from an image using VideoToVideoSDPipeline.
     """
     try:
         if progress_bar:
@@ -43,44 +25,44 @@ def generate_video_from_image(image, duration_seconds=10, progress_bar=None):
         st.write(f"Generated caption: *{caption}*")
         
         if progress_bar:
-            progress_bar.progress(0.3, "Loading LTX-Video model...")
+            progress_bar.progress(0.3, "Loading Video Generation model...")
             
-        # Initialize LTX-Video components
-        processor = AutoProcessor.from_pretrained("Lightricks/ltx-video")
-        model = AutoModel.from_pretrained("Lightricks/ltx-video")
+        # Initialize Video Generation pipeline
+        pipeline = VideoToVideoSDPipeline.from_pretrained(
+            "cerspense/zeroscope_v2_576w",
+            torch_dtype=torch.float16
+        ).to("cuda" if torch.cuda.is_available() else "cpu")
         
         if progress_bar:
             progress_bar.progress(0.4, "Processing image...")
             
-        # Process image
-        inputs = processor(images=image, return_tensors="pt")
+        # Prepare image
+        if image.mode != "RGB":
+            image = image.convert("RGB")
+        image = image.resize((576, 320))  # Resize to model's expected size
         
         if progress_bar:
             progress_bar.progress(0.5, "Generating video frames...")
             
-        # Generate video frames
-        num_frames = duration_seconds * 30  # 30 FPS
-        with torch.no_grad():
-            video_frames = model.generate(
-                **inputs,
-                num_frames=num_frames,
-                num_inference_steps=50,
-                guidance_scale=7.5,
-                prompt=caption,
-            ).videos[0]
+        # Generate video
+        num_frames = duration_seconds * 8  # 8 FPS for this model
+        video_frames = pipeline(
+            image,
+            num_inference_steps=50,
+            num_frames=num_frames,
+            guidance_scale=7.5,
+            prompt=caption,
+        ).videos[0]
         
         if progress_bar:
             progress_bar.progress(0.8, "Creating final video...")
             
-        # Convert frames to numpy arrays
-        frames = [np.array(frame) for frame in video_frames]
-        
         # Create temporary file for video
         with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_file:
             output_path = tmp_file.name
             
-        # Save video using OpenCV
-        save_video_frames(frames, output_path)
+        # Export video frames
+        export_to_video(video_frames, output_path, fps=8)
         
         if progress_bar:
             progress_bar.progress(1.0, "Video generation complete!")
@@ -89,23 +71,25 @@ def generate_video_from_image(image, duration_seconds=10, progress_bar=None):
         
     except Exception as e:
         st.error(f"Error generating video: {str(e)}")
-        raise  # Re-raise the exception to see the full error message
-        return None, None
+        raise
 
 def main():
-    st.set_page_config(page_title="Video Generator", page_icon="🎥")
+    st.set_page_config(page_title="AI Video Generator", page_icon="🎥")
     
-    st.title("🎥 AI Video Generator")
+    st.title("🎥 Video Generator")
     st.write("""
     Upload an image to generate a video with AI-powered motion and transitions.
     The app will automatically generate a caption for your image and use it as inspiration for the video.
     """)
     
+    # Add warning about computational requirements
+    st.warning("Note: Video generation may take several minutes depending on the duration and available computing resources.")
+    
     # File uploader
     uploaded_file = st.file_uploader("Choose an image", type=['png', 'jpg', 'jpeg'])
     
-    # Duration selector
-    duration = st.slider("Video duration (seconds)", min_value=1, max_value=30, value=10)
+    # Duration selector (adjusted for this model's capabilities)
+    duration = st.slider("Video duration (seconds)", min_value=1, max_value=15, value=5)
     
     if uploaded_file is not None:
         # Display uploaded image