Update tools/visit_webpage.py
Browse files- tools/visit_webpage.py +11 -13
tools/visit_webpage.py
CHANGED
@@ -1,45 +1,43 @@
|
|
1 |
from typing import Any, Optional
|
|
|
2 |
from smolagents.tools import Tool
|
3 |
-
import requests
|
4 |
-
import markdownify
|
5 |
-
import smolagents
|
6 |
|
7 |
class VisitWebpageTool(Tool):
|
8 |
name = "visit_webpage"
|
9 |
description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
|
10 |
inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
|
11 |
output_type = "string"
|
12 |
-
|
|
|
|
|
|
|
|
|
13 |
def forward(self, url: str) -> str:
|
14 |
try:
|
15 |
import requests
|
16 |
from markdownify import markdownify
|
17 |
from requests.exceptions import RequestException
|
18 |
-
|
19 |
from smolagents.utils import truncate_content
|
20 |
except ImportError as e:
|
21 |
raise ImportError(
|
22 |
"You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
|
23 |
) from e
|
|
|
24 |
try:
|
25 |
# Send a GET request to the URL with a 20-second timeout
|
26 |
response = requests.get(url, timeout=20)
|
27 |
response.raise_for_status() # Raise an exception for bad status codes
|
28 |
-
|
29 |
# Convert the HTML content to Markdown
|
30 |
markdown_content = markdownify(response.text).strip()
|
31 |
-
|
32 |
# Remove multiple line breaks
|
33 |
markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
|
34 |
-
|
35 |
return truncate_content(markdown_content, 10000)
|
36 |
-
|
37 |
except requests.exceptions.Timeout:
|
38 |
return "The request timed out. Please try again later or check the URL."
|
39 |
except RequestException as e:
|
40 |
return f"Error fetching the webpage: {str(e)}"
|
41 |
except Exception as e:
|
42 |
-
return f"An unexpected error occurred: {str(e)}"
|
43 |
-
|
44 |
-
def __init__(self, *args, **kwargs):
|
45 |
-
self.is_initialized = False
|
|
|
1 |
from typing import Any, Optional
|
2 |
+
import re
|
3 |
from smolagents.tools import Tool
|
|
|
|
|
|
|
4 |
|
5 |
class VisitWebpageTool(Tool):
|
6 |
name = "visit_webpage"
|
7 |
description = "Visits a webpage at the given url and reads its content as a markdown string. Use this to browse webpages."
|
8 |
inputs = {'url': {'type': 'string', 'description': 'The url of the webpage to visit.'}}
|
9 |
output_type = "string"
|
10 |
+
|
11 |
+
def __init__(self, *args, **kwargs):
|
12 |
+
super().__init__(*args, **kwargs)
|
13 |
+
self.is_initialized = True
|
14 |
+
|
15 |
def forward(self, url: str) -> str:
|
16 |
try:
|
17 |
import requests
|
18 |
from markdownify import markdownify
|
19 |
from requests.exceptions import RequestException
|
|
|
20 |
from smolagents.utils import truncate_content
|
21 |
except ImportError as e:
|
22 |
raise ImportError(
|
23 |
"You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
|
24 |
) from e
|
25 |
+
|
26 |
try:
|
27 |
# Send a GET request to the URL with a 20-second timeout
|
28 |
response = requests.get(url, timeout=20)
|
29 |
response.raise_for_status() # Raise an exception for bad status codes
|
30 |
+
|
31 |
# Convert the HTML content to Markdown
|
32 |
markdown_content = markdownify(response.text).strip()
|
33 |
+
|
34 |
# Remove multiple line breaks
|
35 |
markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
|
36 |
+
|
37 |
return truncate_content(markdown_content, 10000)
|
|
|
38 |
except requests.exceptions.Timeout:
|
39 |
return "The request timed out. Please try again later or check the URL."
|
40 |
except RequestException as e:
|
41 |
return f"Error fetching the webpage: {str(e)}"
|
42 |
except Exception as e:
|
43 |
+
return f"An unexpected error occurred: {str(e)}"
|
|
|
|
|
|