Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -1,28 +1,32 @@
|
|
1 |
import os
|
2 |
-
import json
|
3 |
import gradio as gr
|
4 |
import cv2
|
5 |
from google import genai
|
6 |
from google.genai.types import Part
|
7 |
from tenacity import retry, stop_after_attempt, wait_random_exponential
|
8 |
|
9 |
-
# Retrieve API key from environment variables
|
10 |
GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
|
11 |
if not GOOGLE_API_KEY:
|
12 |
raise ValueError("Please set the GOOGLE_API_KEY environment variable.")
|
13 |
|
14 |
-
# Initialize the Gemini API client
|
15 |
client = genai.Client(api_key=GOOGLE_API_KEY)
|
16 |
|
17 |
-
#
|
18 |
-
MODEL_NAME = "gemini-2.0-flash
|
19 |
|
20 |
@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(3))
|
21 |
def call_gemini(video_file: str, prompt: str) -> str:
|
22 |
"""
|
23 |
-
Call the Gemini model with
|
24 |
-
|
25 |
-
|
|
|
|
|
|
|
|
|
|
|
26 |
"""
|
27 |
with open(video_file, "rb") as f:
|
28 |
file_bytes = f.read()
|
@@ -37,17 +41,31 @@ def call_gemini(video_file: str, prompt: str) -> str:
|
|
37 |
|
38 |
def safe_call_gemini(video_file: str, prompt: str) -> str:
|
39 |
"""
|
40 |
-
Wrapper for call_gemini that catches exceptions and returns
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
41 |
"""
|
42 |
try:
|
43 |
return call_gemini(video_file, prompt)
|
44 |
except Exception as e:
|
45 |
-
|
46 |
-
|
|
|
47 |
|
48 |
def hhmmss_to_seconds(time_str: str) -> float:
|
49 |
"""
|
50 |
Convert a HH:MM:SS formatted string into seconds.
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
"""
|
52 |
parts = time_str.strip().split(":")
|
53 |
parts = [float(p) for p in parts]
|
@@ -60,23 +78,28 @@ def hhmmss_to_seconds(time_str: str) -> float:
|
|
60 |
|
61 |
def get_key_frames(video_file: str, summary: str, user_query: str) -> list:
|
62 |
"""
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
|
|
|
|
67 |
|
68 |
-
Returns
|
|
|
69 |
"""
|
70 |
prompt = (
|
71 |
"List the key timestamps in the video and a brief description of the event at that time. "
|
72 |
-
"Output one line per event in the
|
73 |
)
|
74 |
prompt += f" Video Summary: {summary}"
|
75 |
if user_query:
|
76 |
prompt += f" Focus on: {user_query}"
|
77 |
|
78 |
-
# Use the safe call to get a response or fallback text.
|
79 |
key_frames_response = safe_call_gemini(video_file, prompt)
|
|
|
|
|
|
|
80 |
lines = key_frames_response.strip().split("\n")
|
81 |
key_frames = []
|
82 |
for line in lines:
|
@@ -110,13 +133,14 @@ def get_key_frames(video_file: str, summary: str, user_query: str) -> list:
|
|
110 |
|
111 |
def analyze_video(video_file: str, user_query: str) -> (str, list):
|
112 |
"""
|
113 |
-
|
114 |
-
|
115 |
-
|
|
|
|
|
116 |
|
117 |
Returns:
|
118 |
-
|
119 |
-
- A gallery list of key frames (each as a tuple of (image, caption)).
|
120 |
"""
|
121 |
summary_prompt = "Summarize this video."
|
122 |
if user_query:
|
@@ -135,18 +159,28 @@ def analyze_video(video_file: str, user_query: str) -> (str, list):
|
|
135 |
|
136 |
def gradio_interface(video_file, user_query: str) -> (str, list):
|
137 |
"""
|
138 |
-
Gradio interface function
|
139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
140 |
"""
|
141 |
-
if not video_file:
|
142 |
return "Please upload a valid video file.", []
|
|
|
|
|
143 |
return analyze_video(video_file, user_query)
|
144 |
|
|
|
145 |
iface = gr.Interface(
|
146 |
fn=gradio_interface,
|
147 |
inputs=[
|
148 |
gr.Video(label="Upload Video File"),
|
149 |
-
gr.Textbox(label="Analysis Query (optional): guide the focus of the analysis",
|
|
|
150 |
],
|
151 |
outputs=[
|
152 |
gr.Markdown(label="Security & Surveillance Analysis Report"),
|
@@ -154,11 +188,11 @@ iface = gr.Interface(
|
|
154 |
],
|
155 |
title="AI Video Analysis and Summariser Agent",
|
156 |
description=(
|
157 |
-
"This tool uses Google's Gemini 2.0 Flash model
|
158 |
"It returns a brief summary and extracts key frames based on that summary. "
|
159 |
"Provide a video file and, optionally, a query to guide the analysis."
|
160 |
)
|
161 |
)
|
162 |
|
163 |
if __name__ == "__main__":
|
164 |
-
iface.launch()
|
|
|
1 |
import os
|
|
|
2 |
import gradio as gr
|
3 |
import cv2
|
4 |
from google import genai
|
5 |
from google.genai.types import Part
|
6 |
from tenacity import retry, stop_after_attempt, wait_random_exponential
|
7 |
|
8 |
+
# Retrieve API key from environment variables
|
9 |
GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
|
10 |
if not GOOGLE_API_KEY:
|
11 |
raise ValueError("Please set the GOOGLE_API_KEY environment variable.")
|
12 |
|
13 |
+
# Initialize the Gemini API client
|
14 |
client = genai.Client(api_key=GOOGLE_API_KEY)
|
15 |
|
16 |
+
# Define the model name
|
17 |
+
MODEL_NAME = "gemini-2.0-flash"
|
18 |
|
19 |
@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(3))
|
20 |
def call_gemini(video_file: str, prompt: str) -> str:
|
21 |
"""
|
22 |
+
Call the Gemini model with a video file and prompt.
|
23 |
+
|
24 |
+
Args:
|
25 |
+
video_file (str): Path to the video file
|
26 |
+
prompt (str): Text prompt to guide the analysis
|
27 |
+
|
28 |
+
Returns:
|
29 |
+
str: Response text from the Gemini API
|
30 |
"""
|
31 |
with open(video_file, "rb") as f:
|
32 |
file_bytes = f.read()
|
|
|
41 |
|
42 |
def safe_call_gemini(video_file: str, prompt: str) -> str:
|
43 |
"""
|
44 |
+
Wrapper for call_gemini that catches exceptions and returns error messages.
|
45 |
+
|
46 |
+
Args:
|
47 |
+
video_file (str): Path to the video file
|
48 |
+
prompt (str): Text prompt for the API
|
49 |
+
|
50 |
+
Returns:
|
51 |
+
str: API response or error message
|
52 |
"""
|
53 |
try:
|
54 |
return call_gemini(video_file, prompt)
|
55 |
except Exception as e:
|
56 |
+
error_msg = f"Gemini call failed: {str(e)}"
|
57 |
+
print(error_msg)
|
58 |
+
return error_msg
|
59 |
|
60 |
def hhmmss_to_seconds(time_str: str) -> float:
|
61 |
"""
|
62 |
Convert a HH:MM:SS formatted string into seconds.
|
63 |
+
|
64 |
+
Args:
|
65 |
+
time_str (str): Time string in HH:MM:SS format
|
66 |
+
|
67 |
+
Returns:
|
68 |
+
float: Time in seconds
|
69 |
"""
|
70 |
parts = time_str.strip().split(":")
|
71 |
parts = [float(p) for p in parts]
|
|
|
78 |
|
79 |
def get_key_frames(video_file: str, summary: str, user_query: str) -> list:
|
80 |
"""
|
81 |
+
Extract key frames from the video based on timestamps provided by Gemini.
|
82 |
+
|
83 |
+
Args:
|
84 |
+
video_file (str): Path to the video file
|
85 |
+
summary (str): Video summary to provide context
|
86 |
+
user_query (str): Optional user query to focus the analysis
|
87 |
|
88 |
+
Returns:
|
89 |
+
list: List of tuples (image_array, caption)
|
90 |
"""
|
91 |
prompt = (
|
92 |
"List the key timestamps in the video and a brief description of the event at that time. "
|
93 |
+
"Output one line per event in the format: HH:MM:SS - description. Do not include any extra text."
|
94 |
)
|
95 |
prompt += f" Video Summary: {summary}"
|
96 |
if user_query:
|
97 |
prompt += f" Focus on: {user_query}"
|
98 |
|
|
|
99 |
key_frames_response = safe_call_gemini(video_file, prompt)
|
100 |
+
if "Gemini call failed" in key_frames_response:
|
101 |
+
return []
|
102 |
+
|
103 |
lines = key_frames_response.strip().split("\n")
|
104 |
key_frames = []
|
105 |
for line in lines:
|
|
|
133 |
|
134 |
def analyze_video(video_file: str, user_query: str) -> (str, list):
|
135 |
"""
|
136 |
+
Analyze the video and generate a summary and key frames.
|
137 |
+
|
138 |
+
Args:
|
139 |
+
video_file (str): Path to the video file
|
140 |
+
user_query (str): Optional query to guide the analysis
|
141 |
|
142 |
Returns:
|
143 |
+
tuple: (Markdown report, list of key frames)
|
|
|
144 |
"""
|
145 |
summary_prompt = "Summarize this video."
|
146 |
if user_query:
|
|
|
159 |
|
160 |
def gradio_interface(video_file, user_query: str) -> (str, list):
|
161 |
"""
|
162 |
+
Gradio interface function to process video and return results.
|
163 |
+
|
164 |
+
Args:
|
165 |
+
video_file (str): Path to the uploaded video file
|
166 |
+
user_query (str): Optional query to guide analysis
|
167 |
+
|
168 |
+
Returns:
|
169 |
+
tuple: (Markdown report, gallery of key frames)
|
170 |
"""
|
171 |
+
if not video_file or not os.path.exists(video_file):
|
172 |
return "Please upload a valid video file.", []
|
173 |
+
if not video_file.lower().endswith('.mp4'):
|
174 |
+
return "Please upload an MP4 video file.", []
|
175 |
return analyze_video(video_file, user_query)
|
176 |
|
177 |
+
# Define the Gradio interface
|
178 |
iface = gr.Interface(
|
179 |
fn=gradio_interface,
|
180 |
inputs=[
|
181 |
gr.Video(label="Upload Video File"),
|
182 |
+
gr.Textbox(label="Analysis Query (optional): guide the focus of the analysis",
|
183 |
+
placeholder="e.g., focus on unusual movements near the entrance")
|
184 |
],
|
185 |
outputs=[
|
186 |
gr.Markdown(label="Security & Surveillance Analysis Report"),
|
|
|
188 |
],
|
189 |
title="AI Video Analysis and Summariser Agent",
|
190 |
description=(
|
191 |
+
"This tool uses Google's Gemini 2.0 Flash model to analyze an uploaded video. "
|
192 |
"It returns a brief summary and extracts key frames based on that summary. "
|
193 |
"Provide a video file and, optionally, a query to guide the analysis."
|
194 |
)
|
195 |
)
|
196 |
|
197 |
if __name__ == "__main__":
|
198 |
+
iface.launch()
|