Upload folder using huggingface_hub

.env.example

@@ -0,0 +1,21 @@
+OPENAI_ENDPOINT=https://api.openai.com/v1
+OPENAI_API_KEY=
+
+ANTHROPIC_API_KEY=
+
+GOOGLE_API_KEY=
+
+AZURE_OPENAI_ENDPOINT=
+AZURE_OPENAI_API_KEY=
+
+DEEPSEEK_ENDPOINT=https://api.deepseek.com
+DEEPSEEK_API_KEY=
+
+# Set to false to disable anonymized telemetry
+ANONYMIZED_TELEMETRY=true
+
+# LogLevel: Set to debug to enable verbose logging, set to result to get results only. Available: result | debug | info
+BROWSER_USE_LOGGING_LEVEL=info
+
+CHROME_PATH=
+CHROME_USER_DATA=

.gitignore

@@ -0,0 +1,180 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+test_env/
+
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+temp
+tmp
+
+
+.DS_Store
+
+private_example.py
+private_example
+
+browser_cookies.json
+cookies.json
+AgentHistory.json
+cv_04_24.pdf
+AgentHistoryList.json
+*.gif
+.vercel

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

requirements.txt

@@ -0,0 +1,6 @@
+browser-use
+langchain-google-genai
+pyperclip
+gradio
+python-dotenv
+argparse

src/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: __init__.py.py

src/agent/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: __init__.py.py

src/agent/custom_agent.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_agent.py
+
+import asyncio
+import base64
+import io
+import json
+import logging
+import os
+import pdb
+import textwrap
+import time
+import uuid
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Optional, Type, TypeVar
+
+from dotenv import load_dotenv
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import (
+    BaseMessage,
+    SystemMessage,
+)
+from openai import RateLimitError
+from PIL import Image, ImageDraw, ImageFont
+from pydantic import BaseModel, ValidationError
+
+from browser_use.agent.message_manager.service import MessageManager
+from browser_use.agent.prompts import AgentMessagePrompt, SystemPrompt
+from browser_use.agent.service import Agent
+from browser_use.agent.views import (
+    ActionResult,
+    AgentError,
+    AgentHistory,
+    AgentHistoryList,
+    AgentOutput,
+    AgentStepInfo,
+)
+from browser_use.browser.browser import Browser
+from browser_use.browser.context import BrowserContext
+from browser_use.browser.views import BrowserState, BrowserStateHistory
+from browser_use.controller.registry.views import ActionModel
+from browser_use.controller.service import Controller
+from browser_use.dom.history_tree_processor.service import (
+    DOMHistoryElement,
+    HistoryTreeProcessor,
+)
+from browser_use.telemetry.service import ProductTelemetry
+from browser_use.telemetry.views import (
+    AgentEndTelemetryEvent,
+    AgentRunTelemetryEvent,
+    AgentStepErrorTelemetryEvent,
+)
+from browser_use.utils import time_execution_async
+
+from .custom_views import CustomAgentOutput, CustomAgentStepInfo
+from .custom_massage_manager import CustomMassageManager
+
+logger = logging.getLogger(__name__)
+
+
+class CustomAgent(Agent):
+
+    def __init__(
+            self,
+            task: str,
+            llm: BaseChatModel,
+            add_infos: str = '',
+            browser: Browser | None = None,
+            browser_context: BrowserContext | None = None,
+            controller: Controller = Controller(),
+            use_vision: bool = True,
+            save_conversation_path: Optional[str] = None,
+            max_failures: int = 5,
+            retry_delay: int = 10,
+            system_prompt_class: Type[SystemPrompt] = SystemPrompt,
+            max_input_tokens: int = 128000,
+            validate_output: bool = False,
+            include_attributes: list[str] = [
+                'title',
+                'type',
+                'name',
+                'role',
+                'tabindex',
+                'aria-label',
+                'placeholder',
+                'value',
+                'alt',
+                'aria-expanded',
+            ],
+            max_error_length: int = 400,
+            max_actions_per_step: int = 10,
+    ):
+        super().__init__(task, llm, browser, browser_context, controller, use_vision, save_conversation_path,
+                         max_failures, retry_delay, system_prompt_class, max_input_tokens, validate_output,
+                         include_attributes, max_error_length, max_actions_per_step)
+        self.add_infos = add_infos
+        self.message_manager = CustomMassageManager(
+            llm=self.llm,
+            task=self.task,
+            action_descriptions=self.controller.registry.get_prompt_description(),
+            system_prompt_class=self.system_prompt_class,
+            max_input_tokens=self.max_input_tokens,
+            include_attributes=self.include_attributes,
+            max_error_length=self.max_error_length,
+            max_actions_per_step=self.max_actions_per_step,
+        )
+
+    def _setup_action_models(self) -> None:
+        """Setup dynamic action models from controller's registry"""
+        # Get the dynamic action model from controller's registry
+        self.ActionModel = self.controller.registry.create_action_model()
+        # Create output model with the dynamic actions
+        self.AgentOutput = CustomAgentOutput.type_with_custom_actions(self.ActionModel)
+
+    def _log_response(self, response: CustomAgentOutput) -> None:
+        """Log the model's response"""
+        if 'Success' in response.current_state.prev_action_evaluation:
+            emoji = '✅'
+        elif 'Failed' in response.current_state.prev_action_evaluation:
+            emoji = '❌'
+        else:
+            emoji = '🤷'
+
+        logger.info(f'{emoji} Eval: {response.current_state.prev_action_evaluation}')
+        logger.info(f'🧠 New Memory: {response.current_state.important_contents}')
+        logger.info(f'⏳ Task Progress: {response.current_state.completed_contents}')
+        logger.info(f'🤔 Thought: {response.current_state.thought}')
+        logger.info(f'🎯 Summary: {response.current_state.summary}')
+        for i, action in enumerate(response.action):
+            logger.info(
+                f'🛠️  Action {i + 1}/{len(response.action)}: {action.model_dump_json(exclude_unset=True)}'
+            )
+
+    def update_step_info(self, model_output: CustomAgentOutput, step_info: CustomAgentStepInfo = None):
+        """
+        update step info
+        """
+        if step_info is None:
+            return
+
+        step_info.step_number += 1
+        important_contents = model_output.current_state.important_contents
+        if important_contents and 'None' not in important_contents and important_contents not in step_info.memory:
+            step_info.memory += important_contents + '\n'
+
+        completed_contents = model_output.current_state.completed_contents
+        if completed_contents and 'None' not in completed_contents:
+            step_info.task_progress = completed_contents
+
+    @time_execution_async('--get_next_action')
+    async def get_next_action(self, input_messages: list[BaseMessage]) -> AgentOutput:
+        """Get next action from LLM based on current state"""
+
+        ret = self.llm.invoke(input_messages)
+        parsed_json = json.loads(ret.content.replace('```json', '').replace("```", ""))
+        parsed: AgentOutput = self.AgentOutput(**parsed_json)
+        # cut the number of actions to max_actions_per_step
+        parsed.action = parsed.action[: self.max_actions_per_step]
+        self._log_response(parsed)
+        self.n_steps += 1
+
+        return parsed
+
+    @time_execution_async('--step')
+    async def step(self, step_info: Optional[CustomAgentStepInfo] = None) -> None:
+        """Execute one step of the task"""
+        logger.info(f'\n📍 Step {self.n_steps}')
+        state = None
+        model_output = None
+        result: list[ActionResult] = []
+
+        try:
+            state = await self.browser_context.get_state(use_vision=self.use_vision)
+            self.message_manager.add_state_message(state, self._last_result, step_info)
+            input_messages = self.message_manager.get_messages()
+            model_output = await self.get_next_action(input_messages)
+            self.update_step_info(model_output, step_info)
+            logger.info(f'🧠 All Memory: {step_info.memory}')
+            self._save_conversation(input_messages, model_output)
+            self.message_manager._remove_last_state_message()  # we dont want the whole state in the chat history
+            self.message_manager.add_model_output(model_output)
+
+            result: list[ActionResult] = await self.controller.multi_act(
+                model_output.action, self.browser_context
+            )
+            self._last_result = result
+
+            if len(result) > 0 and result[-1].is_done:
+                logger.info(f'📄 Result: {result[-1].extracted_content}')
+
+            self.consecutive_failures = 0
+
+        except Exception as e:
+            result = self._handle_step_error(e)
+            self._last_result = result
+
+        finally:
+            if not result:
+                return
+            for r in result:
+                if r.error:
+                    self.telemetry.capture(
+                        AgentStepErrorTelemetryEvent(
+                            agent_id=self.agent_id,
+                            error=r.error,
+                        )
+                    )
+            if state:
+                self._make_history_item(model_output, state, result)
+
+    async def run(self, max_steps: int = 100) -> AgentHistoryList:
+        """Execute the task with maximum number of steps"""
+        try:
+            logger.info(f'🚀 Starting task: {self.task}')
+
+            self.telemetry.capture(
+                AgentRunTelemetryEvent(
+                    agent_id=self.agent_id,
+                    task=self.task,
+                )
+            )
+
+            step_info = CustomAgentStepInfo(task=self.task,
+                                            add_infos=self.add_infos,
+                                            step_number=1,
+                                            max_steps=max_steps,
+                                            memory='',
+                                            task_progress=''
+                                            )
+
+            for step in range(max_steps):
+                if self._too_many_failures():
+                    break
+
+                await self.step(step_info)
+
+                if self.history.is_done():
+                    if (
+                            self.validate_output and step < max_steps - 1
+                    ):  # if last step, we dont need to validate
+                        if not await self._validate_output():
+                            continue
+
+                    logger.info('✅ Task completed successfully')
+                    break
+            else:
+                logger.info('❌ Failed to complete task in maximum steps')
+
+            return self.history
+
+        finally:
+            self.telemetry.capture(
+                AgentEndTelemetryEvent(
+                    agent_id=self.agent_id,
+                    task=self.task,
+                    success=self.history.is_done(),
+                    steps=len(self.history.history),
+                )
+            )
+            if not self.injected_browser_context:
+                await self.browser_context.close()
+
+            if not self.injected_browser and self.browser:
+                await self.browser.close()

src/agent/custom_massage_manager.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_massage_manager.py
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import List, Optional, Type
+
+from langchain_anthropic import ChatAnthropic
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import (
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+)
+from langchain_openai import ChatOpenAI
+
+from browser_use.agent.message_manager.views import MessageHistory, MessageMetadata
+from browser_use.agent.prompts import AgentMessagePrompt, SystemPrompt
+from browser_use.agent.views import ActionResult, AgentOutput, AgentStepInfo
+from browser_use.browser.views import BrowserState
+from browser_use.agent.message_manager.service import MessageManager
+
+from .custom_prompts import CustomAgentMessagePrompt
+
+logger = logging.getLogger(__name__)
+
+
+class CustomMassageManager(MessageManager):
+    def __init__(
+            self,
+            llm: BaseChatModel,
+            task: str,
+            action_descriptions: str,
+            system_prompt_class: Type[SystemPrompt],
+            max_input_tokens: int = 128000,
+            estimated_tokens_per_character: int = 3,
+            image_tokens: int = 800,
+            include_attributes: list[str] = [],
+            max_error_length: int = 400,
+            max_actions_per_step: int = 10,
+    ):
+        super().__init__(llm, task, action_descriptions, system_prompt_class, max_input_tokens,
+                         estimated_tokens_per_character, image_tokens, include_attributes, max_error_length,
+                         max_actions_per_step)
+
+        # Move Task info to state_message
+        self.history = MessageHistory()
+        self._add_message_with_tokens(self.system_prompt)
+
+    def add_state_message(
+            self,
+            state: BrowserState,
+            result: Optional[List[ActionResult]] = None,
+            step_info: Optional[AgentStepInfo] = None,
+    ) -> None:
+        """Add browser state as human message"""
+
+        # if keep in memory, add to directly to history and add state without result
+        if result:
+            for r in result:
+                if r.include_in_memory:
+                    if r.extracted_content:
+                        msg = HumanMessage(content=str(r.extracted_content))
+                        self._add_message_with_tokens(msg)
+                    if r.error:
+                        msg = HumanMessage(content=str(r.error)[-self.max_error_length:])
+                        self._add_message_with_tokens(msg)
+                    result = None  # if result in history, we dont want to add it again
+
+        # otherwise add state message and result to next message (which will not stay in memory)
+        state_message = CustomAgentMessagePrompt(
+            state,
+            result,
+            include_attributes=self.include_attributes,
+            max_error_length=self.max_error_length,
+            step_info=step_info,
+        ).get_user_message()
+        self._add_message_with_tokens(state_message)

src/agent/custom_prompts.py

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_prompts.py
+
+from datetime import datetime
+from typing import List, Optional
+
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from browser_use.agent.views import ActionResult, AgentStepInfo
+from browser_use.browser.views import BrowserState
+from browser_use.agent.prompts import SystemPrompt, AgentMessagePrompt
+
+from .custom_views import CustomAgentStepInfo
+
+
+class CustomSystemPrompt(SystemPrompt):
+    def important_rules(self) -> str:
+        """
+        Returns the important rules for the agent.
+        """
+        text = """
+    1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
+       {
+         "current_state": {
+           "prev_action_evaluation": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Ignore the action result. The website is the ground truth. Also mention if something unexpected happened like new suggestions in an input field. Shortly state why/why not. Note that the result you output must be consistent with the reasoning you output afterwards. If you consider it to be 'Failed,' you should reflect on this during your thought.",
+           "important_contents": "Output important contents closely related to user\'s instruction or task on the current page. If there is, please output the contents. If not, please output \"None\".",
+           "completed_contents": "Update the input Task Progress. Completed contents is a general summary of the current contents that have been completed. Just summarize the contents that have been actually completed based on the current page and the history operations. Please list each completed item individually, such as: 1. Input username. 2. Input Password. 3. Click confirm button",
+           "thought": "Think about the requirements that have been completed in previous operations and the requirements that need to be completed in the next one operation. If the output of prev_action_evaluation is 'Failed', please reflect and output your reflection here. If you think you have entered the wrong page, consider to go back to the previous page in next action.",
+           "summary": "Please generate a brief natural language description for the operation in next actions based on your Thought."
+         },
+         "action": [
+           {
+             "action_name": {
+               // action-specific parameters
+             }
+           },
+           // ... more actions in sequence
+         ]
+       }
+
+    2. ACTIONS: You can specify multiple actions to be executed in sequence. 
+
+       Common action sequences:
+       - Form filling: [
+           {"input_text": {"index": 1, "text": "username"}},
+           {"input_text": {"index": 2, "text": "password"}},
+           {"click_element": {"index": 3}}
+         ]
+       - Navigation and extraction: [
+           {"open_new_tab": {}},
+           {"go_to_url": {"url": "https://example.com"}},
+           {"extract_page_content": {}}
+         ]
+
+
+    3. ELEMENT INTERACTION:
+       - Only use indexes that exist in the provided element list
+       - Each element has a unique index number (e.g., "33[:]<button>")
+       - Elements marked with "_[:]" are non-interactive (for context only)
+
+    4. NAVIGATION & ERROR HANDLING:
+       - If no suitable elements exist, use other functions to complete the task
+       - If stuck, try alternative approaches
+       - Handle popups/cookies by accepting or closing them
+       - Use scroll to find elements you are looking for
+
+    5. TASK COMPLETION:
+       - If you think all the requirements of user\'s instruction have been completed and no further operation is required, output the done action to terminate the operation process.
+       - Don't hallucinate actions.
+       - If the task requires specific information - make sure to include everything in the done function. This is what the user will see.
+       - If you are running out of steps (current step), think about speeding it up, and ALWAYS use the done action as the last action.
+
+    6. VISUAL CONTEXT:
+       - When an image is provided, use it to understand the page layout
+       - Bounding boxes with labels correspond to element indexes
+       - Each bounding box and its label have the same color
+       - Most often the label is inside the bounding box, on the top right
+       - Visual context helps verify element locations and relationships
+       - sometimes labels overlap, so use the context to verify the correct element
+
+    7. Form filling:
+       - If you fill a input field and your action sequence is interrupted, most often a list with suggestions poped up under the field and you need to first select the right element from the suggestion list.
+
+    8. ACTION SEQUENCING:
+       - Actions are executed in the order they appear in the list 
+       - Each action should logically follow from the previous one
+       - If the page changes after an action, the sequence is interrupted and you get the new state.
+       - If content only disappears the sequence continues.
+       - Only provide the action sequence until you think the page will change.
+       - Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page like saving, extracting, checkboxes...
+       - only use multiple actions if it makes sense. 
+    """
+        text += f'   - use maximum {self.max_actions_per_step} actions per sequence'
+        return text
+
+    def input_format(self) -> str:
+        return """
+    INPUT STRUCTURE:
+    1. Task: The user\'s instructions you need to complete.
+    2. Hints(Optional): Some hints to help you complete the user\'s instructions.
+    3. Memory: Important contents are recorded during historical operations for use in subsequent operations.
+    4. Task Progress: Up to the current page, the content you have completed can be understood as the progress of the task.
+    5. Current URL: The webpage you're currently on
+    6. Available Tabs: List of open browser tabs
+    7. Interactive Elements: List in the format:
+       index[:]<element_type>element_text</element_type>
+       - index: Numeric identifier for interaction
+       - element_type: HTML element type (button, input, etc.)
+       - element_text: Visible text or element description
+
+    Example:
+    33[:]<button>Submit Form</button>
+    _[:] Non-interactive text
+
+
+    Notes:
+    - Only elements with numeric indexes are interactive
+    - _[:] elements provide context but cannot be interacted with
+    """
+
+    def get_system_message(self) -> SystemMessage:
+        """
+        Get the system prompt for the agent.
+
+        Returns:
+            str: Formatted system prompt
+        """
+        time_str = self.current_date.strftime('%Y-%m-%d %H:%M')
+
+        AGENT_PROMPT = f"""You are a precise browser automation agent that interacts with websites through structured commands. Your role is to:
+    1. Analyze the provided webpage elements and structure
+    2. Plan a sequence of actions to accomplish the given task
+    3. Respond with valid JSON containing your action sequence and state assessment
+
+    Current date and time: {time_str}
+
+    {self.input_format()}
+
+    {self.important_rules()}
+
+    Functions:
+    {self.default_action_description}
+
+    Remember: Your responses must be valid JSON matching the specified format. Each action in the sequence must be valid."""
+        return SystemMessage(content=AGENT_PROMPT)
+
+
+class CustomAgentMessagePrompt:
+    def __init__(
+            self,
+            state: BrowserState,
+            result: Optional[List[ActionResult]] = None,
+            include_attributes: list[str] = [],
+            max_error_length: int = 400,
+            step_info: Optional[CustomAgentStepInfo] = None,
+    ):
+        self.state = state
+        self.result = result
+        self.max_error_length = max_error_length
+        self.include_attributes = include_attributes
+        self.step_info = step_info
+
+    def get_user_message(self) -> HumanMessage:
+        state_description = f"""
+    1. Task: {self.step_info.task}
+    2. Hints(Optional): 
+    {self.step_info.add_infos}
+    3. Memory: 
+    {self.step_info.memory}
+    4. Task Progress: 
+    {self.step_info.task_progress}
+    5. Current url: {self.state.url}
+    6. Available tabs:
+    {self.state.tabs}
+    7. Interactive elements:
+    {self.state.element_tree.clickable_elements_to_string(include_attributes=self.include_attributes)}
+            """
+
+        if self.result:
+            for i, result in enumerate(self.result):
+                if result.extracted_content:
+                    state_description += (
+                        f'\nResult of action {i + 1}/{len(self.result)}: {result.extracted_content}'
+                    )
+                if result.error:
+                    # only use last 300 characters of error
+                    error = result.error[-self.max_error_length:]
+                    state_description += f'\nError of action {i + 1}/{len(self.result)}: ...{error}'
+
+        if self.state.screenshot:
+            # Format message for vision model
+            return HumanMessage(
+                content=[
+                    {'type': 'text', 'text': state_description},
+                    {
+                        'type': 'image_url',
+                        'image_url': {'url': f'data:image/png;base64,{self.state.screenshot}'},
+                    },
+                ]
+            )
+
+        return HumanMessage(content=state_description)

src/agent/custom_views.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_views.py
+
+from dataclasses import dataclass
+from typing import Type
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, create_model
+from browser_use.controller.registry.views import ActionModel
+from browser_use.agent.views import AgentOutput
+
+
+@dataclass
+class CustomAgentStepInfo:
+    step_number: int
+    max_steps: int
+    task: str
+    add_infos: str
+    memory: str
+    task_progress: str
+
+
+class CustomAgentBrain(BaseModel):
+    """Current state of the agent"""
+
+    prev_action_evaluation: str
+    important_contents: str
+    completed_contents: str
+    thought: str
+    summary: str
+
+
+class CustomAgentOutput(AgentOutput):
+    """Output model for agent
+
+    @dev note: this model is extended with custom actions in AgentService. You can also use some fields that are not in this model as provided by the linter, as long as they are registered in the DynamicActions model.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    current_state: CustomAgentBrain
+    action: list[ActionModel]
+
+    @staticmethod
+    def type_with_custom_actions(custom_actions: Type[ActionModel]) -> Type['CustomAgentOutput']:
+        """Extend actions with custom actions"""
+        return create_model(
+            'AgentOutput',
+            __base__=CustomAgentOutput,
+            action=(list[custom_actions], Field(...)),  # Properly annotated field with no default
+            __module__=CustomAgentOutput.__module__,
+        )

src/browser/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: __init__.py.py

src/browser/custom_browser.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_browser.py
+
+import logging
+from browser_use.browser.browser import Browser, BrowserConfig
+from browser_use.browser.context import BrowserContextConfig, BrowserContext
+from .custom_context import CustomBrowserContext
+
+logger = logging.getLogger(__name__)
+
+class CustomBrowser(Browser):
+    async def new_context(
+        self, 
+        config: BrowserContextConfig = BrowserContextConfig(),
+        context=None
+    ) -> BrowserContext:
+        """Create a browser context with custom implementation"""
+        # First get/create the underlying Playwright browser
+        playwright_browser = await self.get_playwright_browser()
+        
+        return CustomBrowserContext(
+            browser=self,  # Pass self instead of playwright browser
+            config=config,
+            context=context
+        )
+
+    async def get_playwright_browser(self):
+        """Ensure we have a Playwright browser instance"""
+        if not self.playwright_browser:
+            await self._init()
+        return self.playwright_browser

src/browser/custom_context.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: custom_context.py
+
+import asyncio
+import base64
+import json
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from playwright.async_api import Browser as PlaywrightBrowser, Page, BrowserContext as PlaywrightContext
+from browser_use.browser.context import BrowserContext, BrowserContextConfig
+
+if TYPE_CHECKING:
+    from .custom_browser import CustomBrowser
+
+logger = logging.getLogger(__name__)
+
+class CustomBrowserContext(BrowserContext):
+
+    def __init__(
+            self,
+            browser: 'CustomBrowser',  # Forward declaration for CustomBrowser
+            config: BrowserContextConfig = BrowserContextConfig(),
+            context: PlaywrightContext = None
+    ):
+        super().__init__(browser=browser, config=config)  # Add proper inheritance
+        self._impl_context = context  # Rename to avoid confusion
+        self._page = None
+        self.session = None  # Add session attribute
+
+    @property
+    def impl_context(self) -> PlaywrightContext:
+        """Returns the underlying Playwright context implementation"""
+        return self._impl_context
+
+    async def _create_context(self, config: BrowserContextConfig = None):
+        """Creates a new browser context"""
+        if self._impl_context:
+            return self._impl_context
+
+        # Get the Playwright browser from our custom browser
+        pw_browser = await self.browser.get_playwright_browser()
+        
+        context_args = {
+            'viewport': self.config.browser_window_size,
+            'no_viewport': False, 
+            'bypass_csp': self.config.disable_security,
+            'ignore_https_errors': self.config.disable_security
+        }
+        
+        if self.config.save_recording_path:
+            context_args.update({
+                'record_video_dir': self.config.save_recording_path,
+                'record_video_size': self.config.browser_window_size
+            })
+
+        self._impl_context = await pw_browser.new_context(**context_args)
+        
+        # Create an initial page
+        self._page = await self._impl_context.new_page()
+        await self._page.goto('about:blank')  # Ensure page is ready
+        
+        return self._impl_context
+
+    async def new_page(self) -> Page:
+        """Creates and returns a new page in this context"""
+        if not self._impl_context:
+            await self._create_context()
+        return await self._impl_context.new_page()
+
+    async def __aenter__(self):
+        if not self._impl_context:
+            await self._create_context()
+        return self
+
+    async def __aexit__(self, *args):
+        if self._impl_context:
+            await self._impl_context.close()
+            self._impl_context = None
+
+    @property
+    def pages(self):
+        """Returns list of pages in context"""
+        return self._impl_context.pages if self._impl_context else []
+
+    async def get_state(self, **kwargs):
+        if self._impl_context:
+            # pages() is a synchronous property, not an async method:
+            pages = self._impl_context.pages
+            if pages:
+                return await super().get_state(**kwargs)
+        return None
+
+    async def get_pages(self):
+        """Get pages in a way that works"""
+        if not self._impl_context:
+            return []
+        # Again, pages() is a property:
+        return self._impl_context.pages

src/controller/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: __init__.py.py

src/controller/custom_controller.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: custom_action.py
+
+import pyperclip
+
+from browser_use.controller.service import Controller
+from browser_use.agent.views import ActionResult
+from browser_use.browser.context import BrowserContext
+
+
+class CustomController(Controller):
+    def __init__(self):
+        super().__init__()
+        self._register_custom_actions()
+
+    def _register_custom_actions(self):
+        """Register all custom browser actions"""
+
+        @self.registry.action('Copy text to clipboard')
+        def copy_to_clipboard(text: str):
+            pyperclip.copy(text)
+            return ActionResult(extracted_content=text)
+
+        @self.registry.action('Paste text from clipboard', requires_browser=True)
+        async def paste_from_clipboard(browser: BrowserContext):
+            text = pyperclip.paste()
+            # send text to browser
+            page = await browser.get_current_page()
+            await page.keyboard.type(text)
+
+            return ActionResult(extracted_content=text)

src/utils/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: __init__.py.py

src/utils/file_utils.py

@@ -0,0 +1,25 @@
+import os
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+def get_latest_files(directory: str, file_types: list = ['.webm', '.zip']) -> Dict[str, Optional[str]]:
+    """Get the latest recording and trace files"""
+    latest_files = {ext: None for ext in file_types}
+    
+    if not os.path.exists(directory):
+        os.makedirs(directory, exist_ok=True)
+        return latest_files
+
+    for file_type in file_types:
+        try:
+            matches = list(Path(directory).rglob(f"*{file_type}"))
+            if matches:
+                latest = max(matches, key=lambda p: p.stat().st_mtime)
+                # Only return files that are complete (not being written)
+                if time.time() - latest.stat().st_mtime > 1.0:
+                    latest_files[file_type] = str(latest)
+        except Exception as e:
+            print(f"Error getting latest {file_type} file: {e}")
+            
+    return latest_files

src/utils/stream_utils.py

@@ -0,0 +1,45 @@
+import base64
+import asyncio
+from typing import AsyncGenerator
+from playwright.async_api import BrowserContext, Error as PlaywrightError
+
+async def capture_screenshot(browser_context: BrowserContext) -> str:
+    """Capture and encode a screenshot"""
+    try:
+        # Get the implementation context
+        context = getattr(browser_context, 'impl_context', None)
+        if not context:
+            return "<div>No browser context implementation available</div>"
+            
+        # Get all pages
+        all_pages = context.pages
+        if not all_pages:
+            return "<div>Waiting for page to be available...</div>"
+        # Use the first page
+        page = all_pages[1]
+        try:
+            screenshot = await page.screenshot(
+                type='jpeg',
+                quality=75,
+                scale="css"
+            )
+            encoded = base64.b64encode(screenshot).decode('utf-8')
+            return f'<img src="data:image/jpeg;base64,{encoded}" style="width:100%; max-width:1200px; border:1px solid #ccc;">'
+        except Exception as e:
+            return f"<div class='error'>Screenshot failed: {str(e)}</div>"
+    except Exception as e:
+        return f"<div class='error'>Screenshot error: {str(e)}</div>"
+
+async def stream_browser_view(browser_context: BrowserContext) -> AsyncGenerator[str, None]:
+    """Stream browser view to the UI"""
+    try:
+        while True:
+            try:
+                screenshot_html = await capture_screenshot(browser_context)
+                yield screenshot_html
+                await asyncio.sleep(0.2)  # 5 FPS
+            except Exception as e:
+                yield f"<div class='error'>Screenshot error: {str(e)}</div>"
+                await asyncio.sleep(1)  # Wait before retrying
+    except Exception as e:
+        yield f"<div class='error'>Stream error: {str(e)}</div>"

src/utils/utils.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: utils.py
+
+import base64
+import os
+
+from langchain_openai import ChatOpenAI, AzureChatOpenAI
+from langchain_anthropic import ChatAnthropic
+from langchain_google_genai import ChatGoogleGenerativeAI
+
+
+def get_llm_model(provider: str, **kwargs):
+    """
+    获取LLM 模型
+    :param provider: 模型类型
+    :param kwargs:
+    :return:
+    """
+    if provider == 'anthropic':
+        if not kwargs.get("base_url", ""):
+            base_url = "https://api.anthropic.com"
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("ANTHROPIC_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        return ChatAnthropic(
+            model_name=kwargs.get("model_name", 'claude-3-5-sonnet-20240620'),
+            temperature=kwargs.get("temperature", 0.0),
+            base_url=base_url,
+            api_key=api_key
+        )
+    elif provider == 'openai':
+        if not kwargs.get("base_url", ""):
+            base_url = "https://api.openai.com/v1"
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("OPENAI_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        return ChatOpenAI(
+            model=kwargs.get("model_name", 'gpt-4o'),
+            temperature=kwargs.get("temperature", 0.0),
+            base_url=base_url,
+            api_key=api_key
+        )
+    elif provider == 'deepseek':
+        if not kwargs.get("base_url", ""):
+            base_url = os.getenv("DEEPSEEK_ENDPOINT", "")
+        else:
+            base_url = kwargs.get("base_url")
+
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("DEEPSEEK_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+
+        return ChatOpenAI(
+            model=kwargs.get("model_name", 'gpt-4o'),
+            temperature=kwargs.get("temperature", 0.0),
+            base_url=base_url,
+            api_key=api_key
+        )
+    elif provider == 'gemini':
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("GOOGLE_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+        return ChatGoogleGenerativeAI(
+            model=kwargs.get("model_name", 'gemini-2.0-flash-exp'),
+            temperature=kwargs.get("temperature", 0.0),
+            google_api_key=api_key,
+        )
+    elif provider == "azure_openai":
+        if not kwargs.get("base_url", ""):
+            base_url = os.getenv("AZURE_OPENAI_ENDPOINT", "")
+        else:
+            base_url = kwargs.get("base_url")
+        if not kwargs.get("api_key", ""):
+            api_key = os.getenv("AZURE_OPENAI_API_KEY", "")
+        else:
+            api_key = kwargs.get("api_key")
+        return AzureChatOpenAI(
+            model=kwargs.get("model_name", 'gpt-4o'),
+            temperature=kwargs.get("temperature", 0.0),
+            api_version="2024-05-01-preview",
+            azure_endpoint=base_url,
+            api_key=api_key
+        )
+    else:
+        raise ValueError(f'Unsupported provider: {provider}')
+
+
+def encode_image(img_path):
+    if not img_path:
+        return None
+    with open(img_path, "rb") as fin:
+        image_data = base64.b64encode(fin.read()).decode("utf-8")
+    return image_data

tests/test_browser_use.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @ProjectName: browser-use-webui
+# @FileName: test_browser_use.py
+import pdb
+
+from dotenv import load_dotenv
+
+load_dotenv()
+import sys
+
+sys.path.append(".")
+import os
+import sys
+from pprint import pprint
+
+import asyncio
+from browser_use import Agent
+from browser_use.agent.views import AgentHistoryList
+
+from src.utils import utils
+
+
+async def test_browser_use_org():
+    from browser_use.browser.browser import Browser, BrowserConfig
+    from browser_use.browser.context import (
+        BrowserContext,
+        BrowserContextConfig,
+        BrowserContextWindowSize,
+    )
+    llm = utils.get_llm_model(
+        provider="azure_openai",
+        model_name="gpt-4o",
+        temperature=0.8,
+        base_url=os.getenv("AZURE_OPENAI_ENDPOINT", ""),
+        api_key=os.getenv("AZURE_OPENAI_API_KEY", "")
+    )
+
+    window_w, window_h = 1920, 1080
+
+    browser = Browser(
+        config=BrowserConfig(
+            headless=False,
+            disable_security=True,
+            extra_chromium_args=[f'--window-size={window_w},{window_h}'],
+        )
+    )
+    async with await browser.new_context(
+            config=BrowserContextConfig(
+                trace_path='./tmp/traces',
+                save_recording_path="./tmp/record_videos",
+                no_viewport=False,
+                browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
+            )
+    ) as browser_context:
+        agent = Agent(
+            task="go to google.com and type 'OpenAI' click search and give me the first url",
+            llm=llm,
+            browser_context=browser_context,
+        )
+        history: AgentHistoryList = await agent.run(max_steps=10)
+
+        print('Final Result:')
+        pprint(history.final_result(), indent=4)
+
+        print('\nErrors:')
+        pprint(history.errors(), indent=4)
+
+        # e.g. xPaths the model clicked on
+        print('\nModel Outputs:')
+        pprint(history.model_actions(), indent=4)
+
+        print('\nThoughts:')
+        pprint(history.model_thoughts(), indent=4)
+    # close browser
+    await browser.close()
+
+
+async def test_browser_use_custom():
+    from playwright.async_api import async_playwright
+    from browser_use.browser.context import BrowserContextWindowSize
+
+    from src.browser.custom_browser import CustomBrowser, BrowserConfig
+    from src.browser.custom_context import BrowserContext, BrowserContextConfig
+    from src.controller.custom_controller import CustomController
+    from src.agent.custom_agent import CustomAgent
+    from src.agent.custom_prompts import CustomSystemPrompt
+    from src.browser.custom_context import CustomBrowserContext
+
+    window_w, window_h = 1920, 1080
+
+    # llm = utils.get_llm_model(
+    #     provider="azure_openai",
+    #     model_name="gpt-4o",
+    #     temperature=0.8,
+    #     base_url=os.getenv("AZURE_OPENAI_ENDPOINT", ""),
+    #     api_key=os.getenv("AZURE_OPENAI_API_KEY", "")
+    # )
+
+    # llm = utils.get_llm_model(
+    #     provider="gemini",
+    #     model_name="gemini-2.0-flash-exp",
+    #     temperature=1.0,
+    #     api_key=os.getenv("GOOGLE_API_KEY", "")
+    # )
+
+    llm = utils.get_llm_model(
+        provider="deepseek",
+        model_name="deepseek-chat",
+        temperature=0.8
+    )
+
+    controller = CustomController()
+    use_own_browser = False
+    disable_security = True
+    use_vision = False
+    playwright = None
+    browser_context_ = None
+    try:
+        if use_own_browser:
+            playwright = await async_playwright().start()
+            chrome_exe = os.getenv("CHROME_PATH", "")
+            chrome_use_data = os.getenv("CHROME_USER_DATA", "")
+            browser_context_ = await playwright.chromium.launch_persistent_context(
+                user_data_dir=chrome_use_data,
+                executable_path=chrome_exe,
+                no_viewport=False,
+                headless=False,  # 保持浏览器窗口可见
+                user_agent=(
+                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                    '(KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
+                ),
+                java_script_enabled=True,
+                bypass_csp=disable_security,
+                ignore_https_errors=disable_security,
+                record_video_dir="./tmp/record_videos",
+                record_video_size={'width': window_w, 'height': window_h}
+            )
+        else:
+            browser_context_ = None
+
+        browser = CustomBrowser(
+            config=BrowserConfig(
+                headless=False,
+                disable_security=True,
+                extra_chromium_args=[f'--window-size={window_w},{window_h}'],
+            )
+        )
+
+        async with await browser.new_context(
+                config=BrowserContextConfig(
+                    trace_path='./tmp/result_processing',
+                    save_recording_path="./tmp/record_videos",
+                    no_viewport=False,
+                    browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
+                ),
+                context=browser_context_
+        ) as browser_context:
+            agent = CustomAgent(
+                task="go to google.com and type 'OpenAI' click search and give me the first url",
+                add_infos="",  # some hints for llm to complete the task
+                llm=llm,
+                browser_context=browser_context,
+                controller=controller,
+                system_prompt_class=CustomSystemPrompt,
+                use_vision=use_vision
+            )
+            history: AgentHistoryList = await agent.run(max_steps=10)
+
+            print('Final Result:')
+            pprint(history.final_result(), indent=4)
+
+            print('\nErrors:')
+            pprint(history.errors(), indent=4)
+
+            # e.g. xPaths the model clicked on
+            print('\nModel Outputs:')
+            pprint(history.model_actions(), indent=4)
+
+            print('\nThoughts:')
+            pprint(history.model_thoughts(), indent=4)
+            # close browser
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+    finally:
+        # 显式关闭持久化上下文
+        if browser_context_:
+            await browser_context_.close()
+
+        # 关闭 Playwright 对象
+        if playwright:
+            await playwright.stop()
+
+        await browser.close()
+
+
+if __name__ == '__main__':
+    # asyncio.run(test_browser_use_org())
+    asyncio.run(test_browser_use_custom())

tests/test_llm_api.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: test_llm_api.py
+import os
+import pdb
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import sys
+
+sys.path.append(".")
+
+
+def test_openai_model():
+    from langchain_core.messages import HumanMessage
+    from src.utils import utils
+
+    llm = utils.get_llm_model(
+        provider="openai",
+        model_name="gpt-4o",
+        temperature=0.8,
+        base_url=os.getenv("OPENAI_ENDPOINT", ""),
+        api_key=os.getenv("OPENAI_API_KEY", "")
+    )
+    image_path = "assets/examples/test.png"
+    image_data = utils.encode_image(image_path)
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": "describe this image"},
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
+            },
+        ]
+    )
+    ai_msg = llm.invoke([message])
+    print(ai_msg.content)
+
+
+def test_gemini_model():
+    # you need to enable your api key first: https://ai.google.dev/palm_docs/oauth_quickstart
+    from langchain_core.messages import HumanMessage
+    from src.utils import utils
+
+    llm = utils.get_llm_model(
+        provider="gemini",
+        model_name="gemini-2.0-flash-exp",
+        temperature=0.8,
+        api_key=os.getenv("GOOGLE_API_KEY", "")
+    )
+
+    image_path = "assets/examples/test.png"
+    image_data = utils.encode_image(image_path)
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": "describe this image"},
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
+            },
+        ]
+    )
+    ai_msg = llm.invoke([message])
+    print(ai_msg.content)
+
+
+def test_azure_openai_model():
+    from langchain_core.messages import HumanMessage
+    from src.utils import utils
+
+    llm = utils.get_llm_model(
+        provider="azure_openai",
+        model_name="gpt-4o",
+        temperature=0.8,
+        base_url=os.getenv("AZURE_OPENAI_ENDPOINT", ""),
+        api_key=os.getenv("AZURE_OPENAI_API_KEY", "")
+    )
+    image_path = "assets/examples/test.png"
+    image_data = utils.encode_image(image_path)
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": "describe this image"},
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
+            },
+        ]
+    )
+    ai_msg = llm.invoke([message])
+    print(ai_msg.content)
+
+
+def test_deepseek_model():
+    from langchain_core.messages import HumanMessage
+    from src.utils import utils
+
+    llm = utils.get_llm_model(
+        provider="deepseek",
+        model_name="deepseek-chat",
+        temperature=0.8,
+        base_url=os.getenv("DEEPSEEK_ENDPOINT", ""),
+        api_key=os.getenv("DEEPSEEK_API_KEY", "")
+    )
+    pdb.set_trace()
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": "who are you?"}
+        ]
+    )
+    ai_msg = llm.invoke([message])
+    print(ai_msg.content)
+
+
+if __name__ == '__main__':
+    # test_openai_model()
+    # test_gemini_model()
+    # test_azure_openai_model()
+    test_deepseek_model()

tests/test_playwright.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/2
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: test_playwright.py
+import pdb
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def test_connect_browser():
+    import os
+    from playwright.sync_api import sync_playwright
+
+    chrome_exe = os.getenv("CHROME_PATH", "")
+    chrome_use_data = os.getenv("CHROME_USER_DATA", "")
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch_persistent_context(
+            user_data_dir=chrome_use_data,
+            executable_path=chrome_exe,
+            headless=False  # 保持浏览器窗口可见
+        )
+
+        page = browser.new_page()
+        page.goto("https://mail.google.com/mail/u/0/#inbox")
+        page.wait_for_load_state()
+
+        input("按下回车键以关闭浏览器...")
+
+        browser.close()
+
+
+if __name__ == '__main__':
+    test_connect_browser()

webui.py

@@ -0,0 +1,450 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2025/1/1
+# @Author  : wenshao
+# @Email   : [email protected]
+# @Project : browser-use-webui
+# @FileName: webui.py
+from dotenv import load_dotenv
+load_dotenv()
+import argparse
+import gradio as gr
+import os
+import asyncio
+from playwright.async_api import async_playwright
+from browser_use.browser.browser import Browser, BrowserConfig
+from browser_use.browser.context import (
+    BrowserContextConfig,
+    BrowserContextWindowSize,
+)
+from browser_use.agent.service import Agent
+from src.browser.custom_browser import CustomBrowser
+from src.controller.custom_controller import CustomController
+from src.agent.custom_agent import CustomAgent
+from src.agent.custom_prompts import CustomSystemPrompt
+
+from src.utils import utils
+from src.utils.file_utils import get_latest_files
+from src.utils.stream_utils import stream_browser_view, capture_screenshot
+
+
+async def run_browser_agent(
+        agent_type,
+        llm_provider,
+        llm_model_name,
+        llm_temperature,
+        llm_base_url,
+        llm_api_key,
+        use_own_browser,
+        headless,
+        disable_security,
+        window_w,
+        window_h,
+        save_recording_path,
+        task,
+        add_infos,
+        max_steps,
+        use_vision,
+        browser_context=None  # Added optional argument
+):
+    """
+    Runs the browser agent based on user configurations.
+    """
+
+    llm = utils.get_llm_model(
+        provider=llm_provider,
+        model_name=llm_model_name,
+        temperature=llm_temperature,
+        base_url=llm_base_url,
+        api_key=llm_api_key
+    )
+    if agent_type == "org":
+        return await run_org_agent(
+            llm=llm,
+            headless=headless,
+            disable_security=disable_security,
+            window_w=window_w,
+            window_h=window_h,
+            save_recording_path=save_recording_path,
+            task=task,
+            max_steps=max_steps,
+            use_vision=use_vision,
+            browser_context=browser_context  # pass context
+        )
+    elif agent_type == "custom":
+        return await run_custom_agent(
+            llm=llm,
+            use_own_browser=use_own_browser,
+            headless=headless,
+            disable_security=disable_security,
+            window_w=window_w,
+            window_h=window_h,
+            save_recording_path=save_recording_path,
+            task=task,
+            add_infos=add_infos,
+            max_steps=max_steps,
+            use_vision=use_vision,
+            browser_context=browser_context  # pass context
+        )
+    else:
+        raise ValueError(f"Invalid agent type: {agent_type}")
+
+
+async def run_org_agent(
+        llm,
+        headless,
+        disable_security,
+        window_w,
+        window_h,
+        save_recording_path,
+        task,
+        max_steps,
+        use_vision,
+        browser_context=None  # receive context
+):
+    browser = None
+    if browser_context is None:
+        browser = Browser(
+            config=BrowserConfig(
+                headless=False,  # Force non-headless for streaming
+                disable_security=disable_security,
+                extra_chromium_args=[f'--window-size={window_w},{window_h}'],
+            )
+        )
+        async with await browser.new_context(
+                config=BrowserContextConfig(
+                    trace_path='./tmp/traces',
+                    save_recording_path=save_recording_path if save_recording_path else None,
+                    no_viewport=False,
+                    browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
+                )
+        ) as browser_context_in:
+            agent = Agent(
+                task=task,
+                llm=llm,
+                use_vision=use_vision,
+                browser_context=browser_context_in,
+            )
+            history = await agent.run(max_steps=max_steps)
+            
+            final_result = history.final_result()
+            errors = history.errors()
+            model_actions = history.model_actions()
+            model_thoughts = history.model_thoughts()
+        
+        recorded_files = get_latest_files(save_recording_path)
+        
+        await browser.close()
+        return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), recorded_files.get('.zip')
+    else:
+        # Reuse existing context
+        agent = Agent(
+            task=task,
+            llm=llm,
+            use_vision=use_vision,
+            browser_context=browser_context
+        )
+        history = await agent.run(max_steps=max_steps)
+        final_result = history.final_result()
+        errors = history.errors()
+        model_actions = history.model_actions()
+        model_thoughts = history.model_thoughts()
+        recorded_files = get_latest_files(save_recording_path)
+        return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), recorded_files.get('.zip')
+
+
+async def run_custom_agent(
+        llm,
+        use_own_browser,
+        headless,
+        disable_security,
+        window_w,
+        window_h,
+        save_recording_path,
+        task,
+        add_infos,
+        max_steps,
+        use_vision,
+        browser_context=None  # receive context
+):
+    controller = CustomController()
+    playwright = None
+    browser = None
+    try:
+        if use_own_browser:
+            playwright = await async_playwright().start()
+            chrome_exe = os.getenv("CHROME_PATH", "")
+            chrome_use_data = os.getenv("CHROME_USER_DATA", "")
+            browser_context_ = await playwright.chromium.launch_persistent_context(
+                user_data_dir=chrome_use_data,
+                executable_path=chrome_exe,
+                no_viewport=False,
+                headless=headless,  # 保持浏览器窗口可见
+                user_agent=(
+                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                    '(KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
+                ),
+                java_script_enabled=True,
+                bypass_csp=disable_security,
+                ignore_https_errors=disable_security,
+                record_video_dir=save_recording_path if save_recording_path else None,
+                record_video_size={'width': window_w, 'height': window_h}
+            )
+        else:
+            browser_context_ = None
+
+        if browser_context is not None:
+            # Reuse context
+            agent = CustomAgent(
+                task=task,
+                add_infos=add_infos,
+                use_vision=use_vision,
+                llm=llm,
+                browser_context=browser_context,
+                controller=controller,
+                system_prompt_class=CustomSystemPrompt
+            )
+            history = await agent.run(max_steps=max_steps)
+            final_result = history.final_result()
+            errors = history.errors()
+            model_actions = history.model_actions()
+            model_thoughts = history.model_thoughts()
+            recorded_files = get_latest_files(save_recording_path)
+            return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), recorded_files.get('.zip')
+        else:
+            browser = CustomBrowser(
+                config=BrowserConfig(
+                    headless=headless,
+                    disable_security=disable_security,
+                    extra_chromium_args=[f'--window-size={window_w},{window_h}'],
+                )
+            )
+            async with await browser.new_context(
+                    config=BrowserContextConfig(
+                        trace_path='./tmp/result_processing',
+                        save_recording_path=save_recording_path if save_recording_path else None,
+                        no_viewport=False,
+                        browser_window_size=BrowserContextWindowSize(width=window_w, height=window_h),
+                    ),
+                    context=browser_context_
+            ) as browser_context_in:
+                agent = CustomAgent(
+                    task=task,
+                    add_infos=add_infos,
+                    use_vision=use_vision,
+                    llm=llm,
+                    browser_context=browser_context_in,
+                    controller=controller,
+                    system_prompt_class=CustomSystemPrompt
+                )
+                history = await agent.run(max_steps=max_steps)
+
+                final_result = history.final_result()
+                errors = history.errors()
+                model_actions = history.model_actions()
+                model_thoughts = history.model_thoughts()
+                
+                recorded_files = get_latest_files(save_recording_path)
+
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        final_result = ""
+        errors = str(e) + "\n" + traceback.format_exc()
+        model_actions = ""
+        model_thoughts = ""
+        recorded_files = {}
+    finally:
+        # 显式关闭持久化上下文
+        if browser_context_:
+            await browser_context_.close()
+
+        # 关闭 Playwright 对象
+        if playwright:
+            await playwright.stop()
+        if browser:
+            await browser.close()
+    return final_result, errors, model_actions, model_thoughts, recorded_files.get('.webm'), recorded_files.get('.zip')
+
+
+async def run_with_stream(*args):
+    """Wrapper to run agent and handle streaming"""
+    browser = None
+    try:
+        browser = CustomBrowser(config=BrowserConfig(
+            headless=False,
+            disable_security=args[8],
+            extra_chromium_args=[f'--window-size={args[9]},{args[10]}'],
+        ))
+
+        async with await browser.new_context(
+            config=BrowserContextConfig(
+                trace_path='./tmp/traces',
+                save_recording_path=args[11],
+                no_viewport=False,
+                browser_window_size=BrowserContextWindowSize(width=args[9], height=args[10]),
+            )
+        ) as browser_context:
+            # No need to explicitly create page - context creation handles it
+            
+            # Run agent in background
+            agent_task = asyncio.create_task(run_browser_agent(*args, browser_context=browser_context))
+            
+            # Initialize values
+            html_content = "<div>Starting browser...</div>"
+            final_result = errors = model_actions = model_thoughts = ""
+            recording = trace = None
+
+            while not agent_task.done():
+                try:
+                    html_content = await capture_screenshot(browser_context)
+                except Exception as e:
+                    html_content = f"<div class='error'>Screenshot error: {str(e)}</div>"
+                    
+                yield [html_content, final_result, errors, model_actions, model_thoughts, recording, trace]
+                await asyncio.sleep(0.2)
+
+            # Get agent results when done
+            try:
+                result = await agent_task
+                if isinstance(result, tuple) and len(result) == 6:
+                    final_result, errors, model_actions, model_thoughts, recording, trace = result
+                else:
+                    errors = "Unexpected result format from agent"
+            except Exception as e:
+                errors = f"Agent error: {str(e)}"
+
+            yield [
+                html_content,
+                final_result,
+                errors, 
+                model_actions,
+                model_thoughts,
+                recording,
+                trace
+            ]
+
+    except Exception as e:
+        import traceback
+        yield [
+            f"<div class='error'>Browser error: {str(e)}</div>",
+            "",
+            f"Error: {str(e)}\n{traceback.format_exc()}",
+            "",
+            "",
+            None,
+            None
+        ]
+    finally:
+        if browser:
+            await browser.close()
+
+
+def main():
+    # Gradio UI setup
+    with gr.Blocks(title="Browser Use WebUI", theme=gr.themes.Soft(font=[gr.themes.GoogleFont("Plus Jakarta Sans")])) as demo:
+        gr.Markdown("<center><h1>Browser Use WebUI</h1></center>")
+        with gr.Row():
+            agent_type = gr.Radio(["org", "custom"], label="Agent Type", value="custom")
+            max_steps = gr.Number(label="max run steps", value=100)
+            use_vision = gr.Checkbox(label="use vision", value=True)
+        with gr.Row():
+            llm_provider = gr.Dropdown(
+                ["anthropic", "openai", "gemini", "azure_openai", "deepseek"], label="LLM Provider", value="gemini"
+            )
+            llm_model_name = gr.Textbox(label="LLM Model Name", value="gemini-2.0-flash-exp")
+            llm_temperature = gr.Number(label="LLM Temperature", value=1.0)
+        with gr.Row():
+            llm_base_url = gr.Textbox(label="LLM Base URL")
+            llm_api_key = gr.Textbox(label="LLM API Key", type="password")
+
+        with gr.Accordion("Browser Settings", open=False):
+            use_own_browser = gr.Checkbox(label="Use Own Browser", value=False)
+            headless = gr.Checkbox(label="Headless", value=False)
+            disable_security = gr.Checkbox(label="Disable Security", value=True)
+            with gr.Row():
+                window_w = gr.Number(label="Window Width", value=1920)
+                window_h = gr.Number(label="Window Height", value=1080)
+            save_recording_path = gr.Textbox(label="Save Recording Path", placeholder="e.g. ./tmp/record_videos",
+                                             value="./tmp/record_videos")
+        with gr.Accordion("Task Settings", open=True):
+            task = gr.Textbox(label="Task", lines=10,
+                              value="go to google.com and type 'OpenAI' click search and give me the first url")
+            add_infos = gr.Textbox(label="Additional Infos(Optional): Hints to help LLM complete Task", lines=5)
+
+        run_button = gr.Button("Run Agent", variant="primary")
+        with gr.Column():
+            # Add live stream viewer before other components
+            browser_view = gr.HTML(
+                label="Live Browser View",
+                value="<div style='width:100%; height:600px; border:1px solid #ccc; display:flex; align-items:center; justify-content:center;'><p>Waiting for browser session...</p></div>"
+            )
+            final_result_output = gr.Textbox(label="Final Result", lines=5)
+            errors_output = gr.Textbox(label="Errors", lines=5)
+            model_actions_output = gr.Textbox(label="Model Actions", lines=5)
+            model_thoughts_output = gr.Textbox(label="Model Thoughts", lines=5)
+            with gr.Row():
+                recording_file = gr.Video(label="Recording File")  # Changed from gr.File to gr.Video
+                trace_file = gr.File(label="Trace File (ZIP)")
+        
+        # Add a refresh button
+        refresh_button = gr.Button("Refresh Files")
+        
+        def refresh_files():
+            recorded_files = get_latest_files("./tmp/record_videos")
+            return (
+                recorded_files.get('.webm') if recorded_files.get('.webm') else None,
+                recorded_files.get('.zip') if recorded_files.get('.zip') else None
+            )
+        
+        refresh_button.click(
+            fn=refresh_files,
+            inputs=[],
+            outputs=[recording_file, trace_file]
+        )
+
+        run_button.click(
+            fn=run_with_stream,
+            inputs=[
+                agent_type,
+                llm_provider,
+                llm_model_name,
+                llm_temperature,
+                llm_base_url,
+                llm_api_key,
+                use_own_browser,
+                headless,
+                disable_security,
+                window_w,
+                window_h,
+                save_recording_path,
+                task,
+                add_infos,
+                max_steps,
+                use_vision
+            ],
+            outputs=[  # Change from dict to list
+                browser_view,
+                final_result_output,
+                errors_output,
+                model_actions_output,
+                model_thoughts_output,
+                recording_file,
+                trace_file
+            ],
+            queue=True
+        )
+
+    demo.launch(server_name=args.ip, server_port=args.port, share=True)
+
+if __name__ == "__main__":
+
+    # For local development
+    import argparse
+    parser = argparse.ArgumentParser(description="Gradio UI for Browser Agent")
+    parser.add_argument("--ip", type=str, default="127.0.0.1", help="IP address to bind to")
+    parser.add_argument("--port", type=int, default=7788, help="Port to listen on")
+    args = parser.parse_args()
+    main()
+else:
+    # For Vercel deployment
+    main()