Create utils.py

utils.py

@@ -0,0 +1,65 @@
+import whisper
+import os
+import torch
+import clip
+from PIL import Image
+from moviepy.editor import *
+from transformers import pipeline
+from langdetect import detect
+from gtts import gTTS
+import tempfile
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+clip_model, preprocess = clip.load("ViT-B/32", device=device)
+whisper_model = whisper.load_model("small")
+translator = pipeline("translation", model="Helsinki-NLP/opus-mt-mul-en")
+
+def transcribe_audio_segments(audio_path):
+    result = whisper_model.transcribe(audio_path, word_timestamps=True)
+    segments = result["segments"]
+    return segments
+
+def translate_text(text, target_lang="en"):
+    detected = detect(text)
+    if detected != target_lang:
+        translated = translator(text)[0]['translation_text']
+        return translated
+    return text
+
+def match_images(text, image_paths):
+    text = clip.tokenize([text]).to(device)
+    images = [preprocess(Image.open(img_path)).unsqueeze(0).to(device) for img_path in image_paths]
+    images_tensor = torch.cat(images)
+    with torch.no_grad():
+        text_features = clip_model.encode_text(text)
+        image_features = clip_model.encode_image(images_tensor)
+        similarity = (image_features @ text_features.T).squeeze()
+    best_index = similarity.argmax().item()
+    return image_paths[best_index]
+
+def generate_speech(text, lang="en"):
+    tts = gTTS(text, lang=lang)
+    temp_audio = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+    tts.save(temp_audio.name)
+    return temp_audio.name
+
+def create_video_segments(segments, audio_path, image_paths, output_path="final_video.mp4"):
+    clips = []
+    for seg in segments:
+        segment_text = seg['text'].strip()
+        translated = translate_text(segment_text)
+        duration = seg['end'] - seg['start']
+        matched_img = match_images(translated, image_paths)
+        image_clip = ImageClip(matched_img).set_duration(duration)
+
+        txt_clip = TextClip(translated, fontsize=30, color='white', bg_color='black', size=image_clip.size)
+        txt_clip = txt_clip.set_duration(duration).set_position(("center", "bottom"))
+
+        audio = AudioFileClip(audio_path).subclip(seg['start'], seg['end'])
+        final_clip = CompositeVideoClip([image_clip, txt_clip])
+        final_clip = final_clip.set_audio(audio)
+        clips.append(final_clip)
+
+    final_video = concatenate_videoclips(clips, method="compose")
+    final_video.write_videofile(output_path, fps=24)
+    return output_path