Update app.py
Browse files
app.py
CHANGED
@@ -5,38 +5,55 @@ import os
|
|
5 |
import cv2
|
6 |
import subprocess
|
7 |
|
8 |
-
# ---
|
9 |
try:
|
10 |
nlp = spacy.load("en_core_web_sm")
|
11 |
except OSError:
|
12 |
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
|
13 |
nlp = spacy.load("en_core_web_sm")
|
14 |
|
15 |
-
# ---
|
16 |
ASSET_MAP = {
|
17 |
"man": "assets/characters/man.png",
|
18 |
"woman": "assets/characters/woman.png",
|
19 |
"dog": "assets/characters/dog.png",
|
20 |
"park": "assets/backgrounds/park.jpg",
|
21 |
-
"office": "assets/backgrounds/office.jpg"
|
|
|
|
|
|
|
22 |
}
|
23 |
|
24 |
FRAME_FOLDER = "frames"
|
25 |
VIDEO_OUTPUT = "generated_video.mp4"
|
26 |
|
27 |
-
# --- Extract characters
|
28 |
def extract_entities(prompt):
|
29 |
doc = nlp(prompt)
|
30 |
characters = []
|
31 |
scenes = []
|
|
|
|
|
32 |
for ent in doc.ents:
|
33 |
if ent.label_ in ["PERSON", "ORG"]:
|
34 |
characters.append(ent.text.lower())
|
35 |
elif ent.label_ in ["LOC", "GPE", "FAC"]:
|
36 |
scenes.append(ent.text.lower())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
return characters, scenes
|
38 |
|
39 |
-
# --- Compose a single frame ---
|
40 |
def compose_frame(background_path, character_paths, output_path, char_positions=None):
|
41 |
bg = Image.open(background_path).convert('RGBA')
|
42 |
for idx, char_path in enumerate(character_paths):
|
@@ -45,7 +62,7 @@ def compose_frame(background_path, character_paths, output_path, char_positions=
|
|
45 |
bg.paste(char_img, pos, char_img)
|
46 |
bg.save(output_path)
|
47 |
|
48 |
-
# --- Create video from frames ---
|
49 |
def create_video_from_frames(frame_folder, output_path, fps=24):
|
50 |
images = sorted([img for img in os.listdir(frame_folder) if img.endswith(".png")])
|
51 |
if not images:
|
@@ -62,30 +79,29 @@ def create_video_from_frames(frame_folder, output_path, fps=24):
|
|
62 |
# --- Main function triggered by Gradio ---
|
63 |
def generate_video(prompt):
|
64 |
characters, scenes = extract_entities(prompt)
|
65 |
-
|
66 |
-
if not scenes:
|
67 |
-
return None, "No scene detected! Please include a place in your prompt."
|
68 |
|
69 |
os.makedirs(FRAME_FOLDER, exist_ok=True)
|
70 |
|
71 |
bg_path = ASSET_MAP.get(scenes[0], ASSET_MAP["park"])
|
72 |
char_paths = [ASSET_MAP.get(char, ASSET_MAP["man"]) for char in characters]
|
73 |
|
74 |
-
total_frames = 48 #
|
75 |
for i in range(total_frames):
|
76 |
-
positions = [(100 + i*2, 200) for _ in char_paths]
|
77 |
frame_path = os.path.join(FRAME_FOLDER, f"frame_{i:03d}.png")
|
78 |
compose_frame(bg_path, char_paths, frame_path, char_positions=positions)
|
79 |
|
80 |
create_video_from_frames(FRAME_FOLDER, VIDEO_OUTPUT)
|
81 |
-
|
|
|
|
|
82 |
|
83 |
-
# --- Gradio interface ---
|
84 |
iface = gr.Interface(
|
85 |
fn=generate_video,
|
86 |
inputs=gr.Textbox(lines=3, placeholder="Describe your scene here..."),
|
87 |
outputs=[gr.Video(), gr.Textbox()],
|
88 |
-
title="Text to Video AI App"
|
89 |
)
|
90 |
|
91 |
if __name__ == "__main__":
|
|
|
5 |
import cv2
|
6 |
import subprocess
|
7 |
|
8 |
+
# --- Load SpaCy model dynamically (avoids build-time download issues) ---
|
9 |
try:
|
10 |
nlp = spacy.load("en_core_web_sm")
|
11 |
except OSError:
|
12 |
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
|
13 |
nlp = spacy.load("en_core_web_sm")
|
14 |
|
15 |
+
# --- Define asset mapping (character and background files) ---
|
16 |
ASSET_MAP = {
|
17 |
"man": "assets/characters/man.png",
|
18 |
"woman": "assets/characters/woman.png",
|
19 |
"dog": "assets/characters/dog.png",
|
20 |
"park": "assets/backgrounds/park.jpg",
|
21 |
+
"office": "assets/backgrounds/office.jpg",
|
22 |
+
"home": "assets/backgrounds/home.jpg",
|
23 |
+
"school": "assets/backgrounds/school.jpg",
|
24 |
+
"street": "assets/backgrounds/street.jpg"
|
25 |
}
|
26 |
|
27 |
FRAME_FOLDER = "frames"
|
28 |
VIDEO_OUTPUT = "generated_video.mp4"
|
29 |
|
30 |
+
# --- Extract characters and scenes from prompt ---
|
31 |
def extract_entities(prompt):
|
32 |
doc = nlp(prompt)
|
33 |
characters = []
|
34 |
scenes = []
|
35 |
+
|
36 |
+
# Named Entity Recognition
|
37 |
for ent in doc.ents:
|
38 |
if ent.label_ in ["PERSON", "ORG"]:
|
39 |
characters.append(ent.text.lower())
|
40 |
elif ent.label_ in ["LOC", "GPE", "FAC"]:
|
41 |
scenes.append(ent.text.lower())
|
42 |
+
|
43 |
+
# If no scenes found → keyword matching from ASSET_MAP keys
|
44 |
+
if not scenes:
|
45 |
+
for keyword in ASSET_MAP.keys():
|
46 |
+
if keyword in prompt.lower() and keyword in ["park", "office", "home", "school", "street"]:
|
47 |
+
scenes.append(keyword)
|
48 |
+
break
|
49 |
+
|
50 |
+
# If still no scene → fallback default
|
51 |
+
if not scenes:
|
52 |
+
scenes.append("park")
|
53 |
+
|
54 |
return characters, scenes
|
55 |
|
56 |
+
# --- Compose a single image frame ---
|
57 |
def compose_frame(background_path, character_paths, output_path, char_positions=None):
|
58 |
bg = Image.open(background_path).convert('RGBA')
|
59 |
for idx, char_path in enumerate(character_paths):
|
|
|
62 |
bg.paste(char_img, pos, char_img)
|
63 |
bg.save(output_path)
|
64 |
|
65 |
+
# --- Create a video from image frames ---
|
66 |
def create_video_from_frames(frame_folder, output_path, fps=24):
|
67 |
images = sorted([img for img in os.listdir(frame_folder) if img.endswith(".png")])
|
68 |
if not images:
|
|
|
79 |
# --- Main function triggered by Gradio ---
|
80 |
def generate_video(prompt):
|
81 |
characters, scenes = extract_entities(prompt)
|
|
|
|
|
|
|
82 |
|
83 |
os.makedirs(FRAME_FOLDER, exist_ok=True)
|
84 |
|
85 |
bg_path = ASSET_MAP.get(scenes[0], ASSET_MAP["park"])
|
86 |
char_paths = [ASSET_MAP.get(char, ASSET_MAP["man"]) for char in characters]
|
87 |
|
88 |
+
total_frames = 48 # 2 sec @ 24fps; increase to 2880 for 2 min
|
89 |
for i in range(total_frames):
|
90 |
+
positions = [(100 + i*2, 200) for _ in char_paths]
|
91 |
frame_path = os.path.join(FRAME_FOLDER, f"frame_{i:03d}.png")
|
92 |
compose_frame(bg_path, char_paths, frame_path, char_positions=positions)
|
93 |
|
94 |
create_video_from_frames(FRAME_FOLDER, VIDEO_OUTPUT)
|
95 |
+
|
96 |
+
details = f"Characters detected: {characters if characters else 'default'}, Scene: {scenes[0]}"
|
97 |
+
return VIDEO_OUTPUT, details
|
98 |
|
99 |
+
# --- Gradio interface setup ---
|
100 |
iface = gr.Interface(
|
101 |
fn=generate_video,
|
102 |
inputs=gr.Textbox(lines=3, placeholder="Describe your scene here..."),
|
103 |
outputs=[gr.Video(), gr.Textbox()],
|
104 |
+
title="Text to Video AI App (with fallback scenes)"
|
105 |
)
|
106 |
|
107 |
if __name__ == "__main__":
|