utils.py

import whisper
import os
import torch
import clip
from PIL import Image
from moviepy.editor import *
from transformers import pipeline
from langdetect import detect
from gtts import gTTS
import tempfile

device = "cuda" if torch.cuda.is_available() else "cpu"
clip_model, preprocess = clip.load("ViT-B/32", device=device)
whisper_model = whisper.load_model("small")
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-mul-en")

def transcribe_audio_segments(audio_path):
    result = whisper_model.transcribe(audio_path, word_timestamps=True)
    segments = result["segments"]
    return segments

def translate_text(text, target_lang="en"):
    detected = detect(text)
    if detected != target_lang:
        translated = translator(text)[0]['translation_text']
        return translated
    return text

def match_images(text, image_paths):
    text = clip.tokenize([text]).to(device)
    images = [preprocess(Image.open(img_path)).unsqueeze(0).to(device) for img_path in image_paths]
    images_tensor = torch.cat(images)
    with torch.no_grad():
        text_features = clip_model.encode_text(text)
        image_features = clip_model.encode_image(images_tensor)
        similarity = (image_features @ text_features.T).squeeze()
    best_index = similarity.argmax().item()
    return image_paths[best_index]

def generate_speech(text, lang="en"):
    tts = gTTS(text, lang=lang)
    temp_audio = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
    tts.save(temp_audio.name)
    return temp_audio.name

def create_video_segments(segments, audio_path, image_paths, output_path="final_video.mp4"):
    clips = []
    for seg in segments:
        segment_text = seg['text'].strip()
        translated = translate_text(segment_text)
        duration = seg['end'] - seg['start']
        matched_img = match_images(translated, image_paths)
        image_clip = ImageClip(matched_img).set_duration(duration)

        txt_clip = TextClip(translated, fontsize=30, color='white', bg_color='black', size=image_clip.size)
        txt_clip = txt_clip.set_duration(duration).set_position(("center", "bottom"))

        audio = AudioFileClip(audio_path).subclip(seg['start'], seg['end'])
        final_clip = CompositeVideoClip([image_clip, txt_clip])
        final_clip = final_clip.set_audio(audio)
        clips.append(final_clip)

    final_video = concatenate_videoclips(clips, method="compose")
    final_video.write_videofile(output_path, fps=24)
    return output_path