File size: 2,794 Bytes
9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd 9ad8324 83ed3dd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
import torch
import gradio as gr
from PIL import Image
import numpy as np
import os
# Use a pipeline as a high-level helper
from transformers import pipeline
# Set device
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
# Initialize the pipelines
caption_image = pipeline("image-to-text",
model="Salesforce/blip-image-captioning-large",
device=device)
# Using a different TTS model that's more stable
narrator = pipeline("text-to-speech",
model="microsoft/speecht5_tts",
device=device)
def ensure_output_dir():
"""Ensure the output directory exists"""
output_dir = os.path.join(os.path.expanduser("~"), "AudioCaptions")
os.makedirs(output_dir, exist_ok=True)
return output_dir
def generate_audio(text):
"""
Generate audio from text and save it
"""
try:
# Generate the speech
speech = narrator(text)
# Create output directory and file path
output_dir = ensure_output_dir()
output_path = os.path.join(output_dir, "caption_audio.wav")
# Save the audio file
with open(output_path, "wb") as f:
f.write(speech["audio"])
return output_path
except Exception as e:
print(f"Error generating audio: {str(e)}")
raise gr.Error(f"Failed to generate audio: {str(e)}")
def caption_my_image(image):
"""
Generate caption for image and convert it to speech
"""
try:
if image is None:
raise gr.Error("Please upload an image")
# Generate caption
captions = caption_image(images=image)
if not captions or len(captions) == 0:
raise gr.Error("Could not generate caption for this image")
caption_text = captions[0]['generated_text']
print(f"Generated caption: {caption_text}")
# Generate audio from caption
audio_path = generate_audio(caption_text)
return [audio_path, caption_text]
except Exception as e:
print(f"Error in caption_my_image: {str(e)}")
raise gr.Error(f"Failed to process image: {str(e)}")
# Create the Gradio interface
demo = gr.Interface(
fn=caption_my_image,
inputs=[
gr.Image(label="Upload Image", type="pil")
],
outputs=[
gr.Audio(label="Generated Audio"),
gr.Textbox(label="Generated Caption")
],
title="Image Captioning with Audio",
description="""
Upload an image and the application will:
1. Generate a descriptive caption for the image
2. Convert the caption to speech
""",
examples=[],
cache_examples=False
)
if __name__ == "__main__":
demo.launch() |