Update GRADIO_UI.py

GRADIO_UI.py

@@ -4,48 +4,41 @@
 #
 import os
 import re
+import json
 from typing import Optional
+import logging
 
 from smolagents.agent_types import AgentAudio, AgentImage, AgentText, handle_agent_output_types
 from smolagents.agents import ActionStep, MultiStepAgent
 from smolagents.memory import MemoryStep
 from smolagents.utils import _is_package_available
 
+# Set up logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
 
-def pull_messages_from_step(
-    step_log: MemoryStep,
-):
+def pull_messages_from_step(step_log: MemoryStep):
     """Extract ChatMessage objects from agent steps with proper nesting"""
     import gradio as gr
 
     if isinstance(step_log, ActionStep):
-        # Output the step number
         step_number = f"Step {step_log.step_number}" if step_log.step_number is not None else ""
         yield gr.ChatMessage(role="assistant", content=f"**{step_number}**")
 
-        # First yield the thought/reasoning from the LLM
         if hasattr(step_log, "model_output") and step_log.model_output is not None:
-            # Clean up the LLM output
             model_output = step_log.model_output.strip()
-            # Remove any trailing <end_code> and extra backticks, handling multiple possible formats
             model_output = re.sub(r"```\s*<end_code>", "```", model_output)
             model_output = re.sub(r"<end_code>\s*```", "```", model_output)
             model_output = re.sub(r"```\s*\n\s*<end_code>", "```", model_output)
             model_output = model_output.strip()
             yield gr.ChatMessage(role="assistant", content=model_output)
 
-        # For tool calls, create a parent message
         if hasattr(step_log, "tool_calls") and step_log.tool_calls is not None:
             first_tool_call = step_log.tool_calls[0]
             used_code = first_tool_call.name == "python_interpreter"
             parent_id = f"call_{len(step_log.tool_calls)}"
-
-            # Tool call becomes the parent message with timing info
             args = first_tool_call.arguments
-            if isinstance(args, dict):
-                content = str(args.get("answer", str(args)))
-            else:
-                content = str(args).strip()
+            content = str(args.get("answer", str(args))) if isinstance(args, dict) else str(args).strip()
 
             if used_code:
                 content = re.sub(r"```.*?\n", "", content)
@@ -57,48 +50,57 @@ def pull_messages_from_step(
             parent_message_tool = gr.ChatMessage(
                 role="assistant",
                 content=content,
-                metadata={
-                    "title": f"🛠️ Used tool {first_tool_call.name}",
-                    "id": parent_id,
-                    "status": "pending",
-                },
+                metadata={"title": f"🛠️ Used tool {first_tool_call.name}", "id": parent_id, "status": "pending"}
             )
             yield parent_message_tool
 
-            # Nesting execution logs under the tool call if they exist
-            if hasattr(step_log, "observations") and (
-                step_log.observations is not None and step_log.observations.strip()
-            ):
-                log_content = step_log.observations.strip()
+            if hasattr(step_log, "observations") and step_log.observations and step_log.observations.strip():
+                log_content = re.sub(r"^Execution logs:\s*", "", step_log.observations.strip())
                 if log_content:
-                    log_content = re.sub(r"^Execution logs:\s*", "", log_content)
-                    yield gr.ChatMessage(
-                        role="assistant",
-                        content=f"{log_content}",
-                        metadata={"title": "📝 Execution Logs", "parent_id": parent_id, "status": "done"},
-                    )
+                    try:
+                        # Try to parse as JSON for table data
+                        data = json.loads(log_content)
+                        if isinstance(data, list) and data and isinstance(data[0], list):
+                            # Format as markdown table
+                            headers = data[0]
+                            rows = data[1:]
+                            table_md = "| " + " | ".join(headers) + " |\n"
+                            table_md += "| " + " | ".join(["---"] * len(headers)) + " |\n"
+                            for row in rows:
+                                table_md += "| " + " | ".join(str(cell) for cell in row) + " |\n"
+                            yield gr.ChatMessage(
+                                role="assistant",
+                                content=table_md,
+                                metadata={"title": "📊 Table Data", "parent_id": parent_id, "status": "done"}
+                            )
+                        else:
+                            yield gr.ChatMessage(
+                                role="assistant",
+                                content=log_content,
+                                metadata={"title": "📝 Execution Logs", "parent_id": parent_id, "status": "done"}
+                            )
+                    except json.JSONDecodeError:
+                        yield gr.ChatMessage(
+                            role="assistant",
+                            content=log_content,
+                            metadata={"title": "📝 Execution Logs", "parent_id": parent_id, "status": "done"}
+                        )
 
-            # Nesting any errors under the tool call
             if hasattr(step_log, "error") and step_log.error is not None:
                 yield gr.ChatMessage(
                     role="assistant",
                     content=str(step_log.error),
-                    metadata={"title": "💥 Error", "parent_id": parent_id, "status": "done"},
+                    metadata={"title": "💥 Error", "parent_id": parent_id, "status": "done"}
                 )
 
-            # Update parent message metadata to done status without yielding a new message
             parent_message_tool.metadata["status"] = "done"
 
-        # Handle standalone errors but not from tool calls
         elif hasattr(step_log, "error") and step_log.error is not None:
             yield gr.ChatMessage(role="assistant", content=str(step_log.error), metadata={"title": "💥 Error"})
 
-        # Calculate duration and token information
         step_footnote = f"{step_number}"
         if hasattr(step_log, "input_token_count") and hasattr(step_log, "output_token_count"):
-            token_str = (
-                f" | Input-tokens:{step_log.input_token_count:,} | Output-tokens:{step_log.output_token_count:,}"
-            )
+            token_str = f" | Input-tokens:{step_log.input_token_count:,} | Output-tokens:{step_log.output_token_count:,}"
             step_footnote += token_str
         if hasattr(step_log, "duration"):
             step_duration = f" | Duration: {round(float(step_log.duration), 2)}" if step_log.duration else None
@@ -107,85 +109,79 @@ def pull_messages_from_step(
         yield gr.ChatMessage(role="assistant", content=f"{step_footnote}")
         yield gr.ChatMessage(role="assistant", content="-----")
 
-        # Handle screenshot if present in observations
         if hasattr(step_log, "observations") and step_log.observations:
             for line in step_log.observations.split("\n"):
                 if line.startswith("Screenshot saved at:"):
                     screenshot_path = line.replace("Screenshot saved at: ", "").strip()
+                    logger.debug(f"Yielding screenshot: {screenshot_path}")
                     yield gr.ChatMessage(
                         role="assistant",
                         content={"path": screenshot_path, "mime_type": "image/png"},
                         metadata={"title": "📸 Screenshot"}
                     )
+                elif line.endswith("_detected.png"):
+                    yield gr.ChatMessage(
+                        role="assistant",
+                        content={"path": line.strip(), "mime_type": "image/png"},
+                        metadata={"title": "🖼️ Detected Elements"}
+                    )
+                elif line and not line.startswith("Current url:"):
+                    yield gr.ChatMessage(
+                        role="assistant",
+                        content=line,
+                        metadata={"title": "📝 Scraped Text"}
+                    )
 
-
-def stream_to_gradio(
-    agent,
-    task: str,
-    reset_agent_memory: bool = False,
-    additional_args: Optional[dict] = None,
-):
-    """Runs an agent with the given task and streams the messages from the agent as gradio ChatMessages."""
+def stream_to_gradio(agent, task: str, api_key: str = None, reset_agent_memory: bool = False, additional_args: Optional[dict] = None):
     if not _is_package_available("gradio"):
-        raise ModuleNotFoundError(
-            "Please install 'gradio' extra to use the GradioUI: `pip install 'smolagents[gradio]'`"
-        )
+        raise ModuleNotFoundError("Please install 'gradio' extra to use the GradioUI: `pip install 'smolagents[gradio]'`")
     import gradio as gr
 
+    if api_key:
+        os.environ["GOOGLE_API_KEY"] = api_key
+        logger.debug("Using user-provided GOOGLE_API_KEY")
+
     total_input_tokens = 0
     total_output_tokens = 0
 
     for step_log in agent.run(task, stream=True, reset=reset_agent_memory, additional_args=additional_args):
-        if hasattr(agent.model, "last_input_token_count"):
-            total_input_tokens += agent.model.last_input_token_count
-            total_output_tokens += agent.model.last_output_token_count
-            if isinstance(step_log, ActionStep):
-                step_log.input_token_count = agent.model.last_input_token_count
-                step_log.output_token_count = agent.model.last_output_token_count
-
-        for message in pull_messages_from_step(
-            step_log,
-        ):
+        input_tokens = agent.model.last_input_token_count
+        output_tokens = agent.model.last_output_token_count
+        logger.debug(f"Input tokens: {input_tokens}, Output tokens: {output_tokens}")
+        if input_tokens is not None:
+            total_input_tokens += input_tokens
+        if output_tokens is not None:
+            total_output_tokens += output_tokens
+        if isinstance(step_log, ActionStep):
+            step_log.input_token_count = input_tokens if input_tokens is not None else 0
+            step_log.output_token_count = output_tokens if output_tokens is not None else 0
+
+        for message in pull_messages_from_step(step_log):
             yield message
 
     final_answer = step_log
     final_answer = handle_agent_output_types(final_answer)
 
     if isinstance(final_answer, AgentText):
-        yield gr.ChatMessage(
-            role="assistant",
-            content=f"**Final answer:**\n{final_answer.to_string()}\n",
-        )
+        yield gr.ChatMessage(role="assistant", content=f"**Final answer:**\n{final_answer.to_string()}\n")
     elif isinstance(final_answer, AgentImage):
-        yield gr.ChatMessage(
-            role="assistant",
-            content={"path": final_answer.to_string(), "mime_type": "image/png"},
-        )
+        yield gr.ChatMessage(role="assistant", content={"path": final_answer.to_string(), "mime_type": "image/png"})
     elif isinstance(final_answer, AgentAudio):
-        yield gr.ChatMessage(
-            role="assistant",
-            content={"path": final_answer.to_string(), "mime_type": "audio/wav"},
-        )
+        yield gr.ChatMessage(role="assistant", content={"path": final_answer.to_string(), "mime_type": "audio/wav"})
     else:
         yield gr.ChatMessage(role="assistant", content=f"**Final answer:** {str(final_answer)}")
 
-
 class GradioUI:
-    """A one-line interface to launch your agent in Gradio"""
-
     def __init__(self, agent: MultiStepAgent):
         if not _is_package_available("gradio"):
-            raise ModuleNotFoundError(
-                "Please install 'gradio' extra to use the GradioUI: `pip install 'smolagents[gradio]'`"
-            )
+            raise ModuleNotFoundError("Please install 'gradio' extra to use the GradioUI: `pip install 'smolagents[gradio]'`")
         self.agent = agent
 
-    def interact_with_agent(self, prompt, messages):
+    def interact_with_agent(self, prompt, api_key, messages):
         import gradio as gr
-
         messages.append(gr.ChatMessage(role="user", content=prompt))
         yield messages
-        for msg in stream_to_gradio(self.agent, task=prompt, reset_agent_memory=False):
+        for msg in stream_to_gradio(self.agent, task=prompt, api_key=api_key, reset_agent_memory=False):
             messages.append(msg)
             yield messages
         yield messages
@@ -193,7 +189,6 @@ class GradioUI:
     def launch(self, **kwargs):
         import gradio as gr
 
-        # Custom CSS for avatar alignment
         css = """
         .chatbot .avatar-container {
             display: flex !important;
@@ -211,26 +206,21 @@ class GradioUI:
         }
         """
 
-        with gr.Blocks(fill_height=True) as demo:
+        with gr.Blocks(fill_height=True, css=css) as demo:
             stored_messages = gr.State([])
+            gr.Markdown("**Note**: Please provide your own Gemini API key below. The default key may run out of quota.")
+            api_key_input = gr.Textbox(
+                lines=1, label="Gemini API Key (optional)", placeholder="Enter your Gemini API key here", type="password"
+            )
             chatbot = gr.Chatbot(
-                label="Web Navigation Agent",
-                type="messages",
-                avatar_images=(
-                    None,
-                    "./icon.png"
-                ),
-                scale=1,
-                height=600,
+                label="Web Navigation Agent", type="messages",
+                avatar_images=(None, "./icon.png"), scale=1, height=600
             )
-            text_input = gr.Textbox(lines=1, label="Enter URL and request (e.g., Please could you navigate to https://www.federalreserve.gov/releases/h15/ and extract interest rates)")
-            text_input.submit(
-                self.interact_with_agent,
-                [text_input, stored_messages],
-                [chatbot]
+            text_input = gr.Textbox(
+                lines=1, label="Enter URL and request (e.g., navigate to https://en.wikipedia.org/wiki/Nvidia, scrape the infobox table, fill a search form)"
             )
+            text_input.submit(self.interact_with_agent, [text_input, api_key_input, stored_messages], [chatbot])
 
         demo.launch(debug=True, **kwargs)
 
-
 __all__ = ["stream_to_gradio", "GradioUI"]