Function calling (#996)

* Initial work on function calling

* wip

* Add websearch as a tool

* lint

* smol

* Move tools to their own files

* directly answer check

* Add text2img tool

* group tool calls together

* show retry even if no message

* fix trailing urls

* Add image popup

* format

* wip

* clean-up

* better loading indicator

* text colors

* extra example to summarize

* switch default model

* Add fetchUrl tool

* wip

* Add latest gradio

* fix types

* version bump sharp

* Basic tools menu

* working menu

* fix menu positioning

* deps fix

* add deps

* cleanup

* more cleanup

* cleanup

* package cleanups

* moar cleanup

* ui update

* Update ChatMessage.svelte

* upgrade gradio dep

* lint

* refactor and pass results to cohere

* feat: code interpreter tool

* fix: add e2b dependency

* feat: working TGI endpoint with tool results

* feat: allow image model to expand to 90dvw

* fix: dont block on title gen

* bump sharp

* misc cleanup

* fix url fetcher, adjust tool typing

* migrate to new MessageUpdate schema

* fix lint errors

* image editing, pdf upload + parsing

* feat: file preview for non-images

* feat: image prompting, file ui, file migration, many fixes

* feat: multiple files and tool file indices

* feat: add back remote keylogging prevention

* minor nit

* minor

* resolve nits

* feat: use node vm for calculator and improve prompt

* more nits

* feat: add index to tools settings

* chore: update invalid package lock

* feat: bump gradio client to 0.19.4

* feat: move pdf to markdown to huggingchat

* fix: uploaded file width

* feat: prompt the model when no files available

* Feat functions: UI update (#1157)

* details: close on click outside

* drag for tools

* tools ordering

* add link to community discussion

* misc

* calculator listed as last

* file style

* images with different ratios

* page scraper: longer timeout



@Saghen
@mishig25 seems to work better with this

* vertical gap

* Revert "drag for tools"

This reverts commit 8eeed3b77c164325bec8ab4f5827641acb6cc72f.

* chat padding on desktop

* file colors

* larger gap for all messages on xl screens

* feat: update pdf to markdown schema

* fix: uploaded file container width

* fix: tool name check

* fix: cohere endpoint

* fix: use most recent message for files

* feat: allow tools to access previous files

* feat: support all file types on document parser

* fix: use document parser in default tools

* feat: rename url fetcher internal name

* feat: resolve type error

* feat: enable tools on prod command r+

* feat: truncate document markdown

* feat: bump @huggingface/inference

* feat: resolve type errors

* nit

* Feat functions misc update 2 (#1158)

* update examples

a

* add tool indicator

* llama 3 description update

* image quick fix

* mobile

* icon purple

---------

Co-authored-by: Victor Mustar <[email protected]>
Co-authored-by: Liam Dyer <[email protected]>
Co-authored-by: Mishig <[email protected]>

.env

@@ -154,6 +154,7 @@ WEBHOOK_URL_REPORT_ASSISTANT=#provide webhook url to get notified when an assist
 ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to use the app
 
 USAGE_LIMITS=`{}`
+
 ALLOW_INSECURE_COOKIES=false # recommended to keep this to false but set to true if you need to run over http without tls
 METRICS_PORT=
 LOG_LEVEL=info

.eslintrc.cjs

@@ -24,7 +24,7 @@ module.exports = {
 		extraFileExtensions: [".svelte"],
 	},
 	rules: {
-		"no-shadow": ["error"],
+		"require-yield": "off",
 		"@typescript-eslint/no-explicit-any": "error",
 		"@typescript-eslint/no-non-null-assertion": "error",
 		"@typescript-eslint/no-unused-vars": [

chart/env/prod.yaml

@@ -39,6 +39,7 @@ envVars:
         "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus",
         "websiteUrl": "https://docs.cohere.com/docs/command-r-plus",
         "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png",
+        "tools": true,
         "parameters": {
           "stop": ["<|END_OF_TURN_TOKEN|>"],
           "truncate" : 28672,
@@ -47,20 +48,20 @@ envVars:
         },
         "promptExamples" : [
           {
-            "title": "Write an email from bullet list",
-            "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
+            "title": "Generate a mouse portrait",
+            "prompt": "Generate the portrait of a scientific mouse in its laboratory."
+          }, {
+            "title": "Review a pull request",
+            "prompt": "Review this pull request: https://github.com/huggingface/chat-ui/pull/1131/files"
           }, {
             "title": "Code a snake game",
             "prompt": "Code a basic snake game in python, give explanations for each step."
-          }, {
-            "title": "Assist in a task",
-            "prompt": "How do I make a delicious lemon cheesecake?"
           }
         ]
       },
       {
         "name" : "meta-llama/Meta-Llama-3-70B-Instruct",
-        "description": "Generation over generation, Meta Llama 3 demonstrates state-of-the-art performance on a wide range of industry benchmarks and offers new capabilities, including improved reasoning.",
+        "description": "Meta Llama 3 delivers top performance on various benchmarks and introduces new features like better reasoning.",
         "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png",
         "modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
         "websiteUrl": "https://llama.meta.com/llama3/",

package-lock.json

@@ -9,8 +9,9 @@
 			"version": "0.8.4",
 			"dependencies": {
 				"@cliqz/adblocker-playwright": "^1.27.2",
+				"@gradio/client": "^0.19.4",
 				"@huggingface/hub": "^0.5.1",
-				"@huggingface/inference": "^2.6.3",
+				"@huggingface/inference": "^2.7.0",
 				"@iconify-json/bi": "^1.1.21",
 				"@playwright/browser-chromium": "^1.43.1",
 				"@resvg/resvg-js": "^2.6.2",
@@ -42,7 +43,7 @@
 				"satori-html": "^0.3.2",
 				"sbd": "^1.0.19",
 				"serpapi": "^1.1.1",
-				"sharp": "^0.33.3",
+				"sharp": "^0.33.4",
 				"tailwind-scrollbar": "^3.0.0",
 				"tailwindcss": "^3.4.0",
 				"uuid": "^9.0.1",
@@ -174,6 +175,22 @@
 				"google-auth-library": "^9.4.2"
 			}
 		},
+		"node_modules/@bundled-es-modules/cookie": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.0.tgz",
+			"integrity": "sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==",
+			"dependencies": {
+				"cookie": "^0.5.0"
+			}
+		},
+		"node_modules/@bundled-es-modules/statuses": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz",
+			"integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==",
+			"dependencies": {
+				"statuses": "^2.0.1"
+			}
+		},
 		"node_modules/@cliqz/adblocker": {
 			"version": "1.27.2",
 			"resolved": "https://registry.npmjs.org/@cliqz/adblocker/-/adblocker-1.27.2.tgz",
@@ -706,6 +723,23 @@
 				"node": ">=18.0.0"
 			}
 		},
+		"node_modules/@gradio/client": {
+			"version": "0.19.4",
+			"resolved": "https://registry.npmjs.org/@gradio/client/-/client-0.19.4.tgz",
+			"integrity": "sha512-O7bSkgoGL7Fe180UTB9IF9ZlIMdiIhWkInE22NDCsH3Lyt7QdOxDSRKZUEYTPELUIBrejWiuQ1ADNJQzDTBvBg==",
+			"dependencies": {
+				"@types/eventsource": "^1.1.15",
+				"bufferutil": "^4.0.7",
+				"eventsource": "^2.0.2",
+				"msw": "^2.2.1",
+				"semiver": "^1.1.0",
+				"typescript": "^5.0.0",
+				"ws": "^8.13.0"
+			},
+			"engines": {
+				"node": ">=18.0.0"
+			}
+		},
 		"node_modules/@huggingface/hub": {
 			"version": "0.5.1",
 			"resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.5.1.tgz",
@@ -718,9 +752,12 @@
 			}
 		},
 		"node_modules/@huggingface/inference": {
-			"version": "2.6.3",
-			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.6.3.tgz",
-			"integrity": "sha512-KK6xNrEldjjopiGqwaBCkA+4tEyuIz0qHsD5SVYaQ65HSlmBbntJieSw4NRWT+S5bK/Bf/GFCixW0NshAOcBqA==",
+			"version": "2.7.0",
+			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.7.0.tgz",
+			"integrity": "sha512-u7Fn637Q3f7nUB1tajM4CgzhvoFQkOQr5W5Fm+2wT9ETgGoLBh25BLlYPTJRjAd2WY01s71v0lqAwNvHHCc3mg==",
+			"dependencies": {
+				"@huggingface/tasks": "^0.10.0"
+			},
 			"engines": {
 				"node": ">=18"
 			}
@@ -733,6 +770,11 @@
 				"node": ">=18"
 			}
 		},
+		"node_modules/@huggingface/tasks": {
+			"version": "0.10.8",
+			"resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.8.tgz",
+			"integrity": "sha512-oIp9912FwByyyyxkB/CIiW203QxPeYzBG7dydLmqZdcW0ma0if1uklGumqkJ6y/rJlv7AR/jJ9wbee0Wl5YZTw=="
+		},
 		"node_modules/@humanwhocodes/config-array": {
 			"version": "0.11.8",
 			"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz",
@@ -812,9 +854,9 @@
 			}
 		},
 		"node_modules/@img/sharp-darwin-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz",
-			"integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.4.tgz",
+			"integrity": "sha512-p0suNqXufJs9t3RqLBO6vvrgr5OhgbWp76s5gTRvdmxmuv9E1rcaqGUsl3l4mKVmXPkTkTErXediAui4x+8PSA==",
 			"cpu": [
 				"arm64"
 			],
@@ -837,9 +879,9 @@
 			}
 		},
 		"node_modules/@img/sharp-darwin-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz",
-			"integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.4.tgz",
+			"integrity": "sha512-0l7yRObwtTi82Z6ebVI2PnHT8EB2NxBgpK2MiKJZJ7cz32R4lxd001ecMhzzsZig3Yv9oclvqqdV93jo9hy+Dw==",
 			"cpu": [
 				"x64"
 			],
@@ -1030,9 +1072,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-arm": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz",
-			"integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.4.tgz",
+			"integrity": "sha512-RUgBD1c0+gCYZGCCe6mMdTiOFS0Zc/XrN0fYd6hISIKcDUbAW5NtSQW9g/powkrXYm6Vzwd6y+fqmExDuCdHNQ==",
 			"cpu": [
 				"arm"
 			],
@@ -1055,9 +1097,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz",
-			"integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.4.tgz",
+			"integrity": "sha512-2800clwVg1ZQtxwSoTlHvtm9ObgAax7V6MTAB/hDT945Tfyy3hVkmiHpeLPCKYqYR1Gcmv1uDZ3a4OFwkdBL7Q==",
 			"cpu": [
 				"arm64"
 			],
@@ -1080,9 +1122,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-s390x": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz",
-			"integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.4.tgz",
+			"integrity": "sha512-h3RAL3siQoyzSoH36tUeS0PDmb5wINKGYzcLB5C6DIiAn2F3udeFAum+gj8IbA/82+8RGCTn7XW8WTFnqag4tQ==",
 			"cpu": [
 				"s390x"
 			],
@@ -1091,7 +1133,7 @@
 				"linux"
 			],
 			"engines": {
-				"glibc": ">=2.28",
+				"glibc": ">=2.31",
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0",
 				"npm": ">=9.6.5",
 				"pnpm": ">=7.1.0",
@@ -1105,9 +1147,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz",
-			"integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.4.tgz",
+			"integrity": "sha512-GoR++s0XW9DGVi8SUGQ/U4AeIzLdNjHka6jidVwapQ/JebGVQIpi52OdyxCNVRE++n1FCLzjDovJNozif7w/Aw==",
 			"cpu": [
 				"x64"
 			],
@@ -1130,9 +1172,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-arm64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz",
-			"integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.4.tgz",
+			"integrity": "sha512-nhr1yC3BlVrKDTl6cO12gTpXMl4ITBUZieehFvMntlCXFzH2bvKG76tBL2Y/OqhupZt81pR7R+Q5YhJxW0rGgQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -1155,9 +1197,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz",
-			"integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.4.tgz",
+			"integrity": "sha512-uCPTku0zwqDmZEOi4ILyGdmW76tH7dm8kKlOIV1XC5cLyJ71ENAAqarOHQh0RLfpIpbV5KOpXzdU6XkJtS0daw==",
 			"cpu": [
 				"x64"
 			],
@@ -1180,15 +1222,15 @@
 			}
 		},
 		"node_modules/@img/sharp-wasm32": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz",
-			"integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.4.tgz",
+			"integrity": "sha512-Bmmauh4sXUsUqkleQahpdNXKvo+wa1V9KhT2pDA4VJGKwnKMJXiSTGphn0gnJrlooda0QxCtXc6RX1XAU6hMnQ==",
 			"cpu": [
 				"wasm32"
 			],
 			"optional": true,
 			"dependencies": {
-				"@emnapi/runtime": "^1.1.0"
+				"@emnapi/runtime": "^1.1.1"
 			},
 			"engines": {
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0",
@@ -1201,9 +1243,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-ia32": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz",
-			"integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.4.tgz",
+			"integrity": "sha512-99SJ91XzUhYHbx7uhK3+9Lf7+LjwMGQZMDlO/E/YVJ7Nc3lyDFZPGhjwiYdctoH2BOzW9+TnfqcaMKt0jHLdqw==",
 			"cpu": [
 				"ia32"
 			],
@@ -1222,9 +1264,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-x64": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz",
-			"integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.4.tgz",
+			"integrity": "sha512-3QLocdTRVIrFNye5YocZl+KKpYKP+fksi1QhmOArgx7GyhIbQp/WrJRu176jm8IxromS7RIkzMiMINVdBtC8Aw==",
 			"cpu": [
 				"x64"
 			],
@@ -1242,6 +1284,76 @@
 				"url": "https://opencollective.com/libvips"
 			}
 		},
+		"node_modules/@inquirer/confirm": {
+			"version": "3.1.7",
+			"resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.7.tgz",
+			"integrity": "sha512-BZjjj19W8gnh5UGFTdP5ZxpgMNRjy03Dzq3k28sB2MDlEUFrcyTkMEoGgvBmGpUw0vNBoCJkTcbHZ3e9tb+d+w==",
+			"dependencies": {
+				"@inquirer/core": "^8.2.0",
+				"@inquirer/type": "^1.3.1"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@inquirer/core": {
+			"version": "8.2.0",
+			"resolved": "https://registry.npmjs.org/@inquirer/core/-/core-8.2.0.tgz",
+			"integrity": "sha512-pexNF9j2orvMMTgoQ/uKOw8V6/R7x/sIDwRwXRhl4i0pPSh6paRzFehpFKpfMbqix1/+gzCekhYTmVbQpWkVjQ==",
+			"dependencies": {
+				"@inquirer/figures": "^1.0.1",
+				"@inquirer/type": "^1.3.1",
+				"@types/mute-stream": "^0.0.4",
+				"@types/node": "^20.12.11",
+				"@types/wrap-ansi": "^3.0.0",
+				"ansi-escapes": "^4.3.2",
+				"chalk": "^4.1.2",
+				"cli-spinners": "^2.9.2",
+				"cli-width": "^4.1.0",
+				"mute-stream": "^1.0.0",
+				"signal-exit": "^4.1.0",
+				"strip-ansi": "^6.0.1",
+				"wrap-ansi": "^6.2.0"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@inquirer/core/node_modules/@types/node": {
+			"version": "20.12.12",
+			"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz",
+			"integrity": "sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==",
+			"dependencies": {
+				"undici-types": "~5.26.4"
+			}
+		},
+		"node_modules/@inquirer/core/node_modules/signal-exit": {
+			"version": "4.1.0",
+			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
+			"integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
+			"engines": {
+				"node": ">=14"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/isaacs"
+			}
+		},
+		"node_modules/@inquirer/figures": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.1.tgz",
+			"integrity": "sha512-mtup3wVKia3ZwULPHcbs4Mor8Voi+iIXEWD7wCNbIO6lYR62oPCTQyrddi5OMYVXHzeCSoneZwJuS8sBvlEwDw==",
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@inquirer/type": {
+			"version": "1.3.1",
+			"resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.3.1.tgz",
+			"integrity": "sha512-Pe3PFccjPVJV1vtlfVvm9OnlbxqdnP5QcscFEFEnK5quChf1ufZtM0r8mR5ToWHMxZOh0s8o/qp9ANGRTo/DAw==",
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/@jridgewell/gen-mapping": {
 			"version": "0.3.3",
 			"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
@@ -1294,6 +1406,30 @@
 				"sparse-bitfield": "^3.0.3"
 			}
 		},
+		"node_modules/@mswjs/cookies": {
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-1.1.0.tgz",
+			"integrity": "sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==",
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@mswjs/interceptors": {
+			"version": "0.29.1",
+			"resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz",
+			"integrity": "sha512-3rDakgJZ77+RiQUuSK69t1F0m8BQKA8Vh5DCS5V0DWvNY67zob2JhhQrhCO0AKLGINTRSFd1tBaHcJTkhefoSw==",
+			"dependencies": {
+				"@open-draft/deferred-promise": "^2.2.0",
+				"@open-draft/logger": "^0.3.0",
+				"@open-draft/until": "^2.0.0",
+				"is-node-process": "^1.2.0",
+				"outvariant": "^1.2.1",
+				"strict-event-emitter": "^0.5.1"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/@nodelib/fs.scandir": {
 			"version": "2.1.5",
 			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -1326,6 +1462,25 @@
 				"node": ">= 8"
 			}
 		},
+		"node_modules/@open-draft/deferred-promise": {
+			"version": "2.2.0",
+			"resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz",
+			"integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA=="
+		},
+		"node_modules/@open-draft/logger": {
+			"version": "0.3.0",
+			"resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz",
+			"integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==",
+			"dependencies": {
+				"is-node-process": "^1.2.0",
+				"outvariant": "^1.4.0"
+			}
+		},
+		"node_modules/@open-draft/until": {
+			"version": "2.1.0",
+			"resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz",
+			"integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg=="
+		},
 		"node_modules/@opentelemetry/api": {
 			"version": "1.8.0",
 			"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.8.0.tgz",
@@ -2168,6 +2323,11 @@
 			"integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==",
 			"dev": true
 		},
+		"node_modules/@types/eventsource": {
+			"version": "1.1.15",
+			"resolved": "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.15.tgz",
+			"integrity": "sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA=="
+		},
 		"node_modules/@types/express": {
 			"version": "4.17.21",
 			"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz",
@@ -2266,6 +2426,14 @@
 			"integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
 			"dev": true
 		},
+		"node_modules/@types/mute-stream": {
+			"version": "0.0.4",
+			"resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz",
+			"integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==",
+			"dependencies": {
+				"@types/node": "*"
+			}
+		},
 		"node_modules/@types/node": {
 			"version": "18.13.0",
 			"resolved": "https://registry.npmjs.org/@types/node/-/node-18.13.0.tgz",
@@ -2356,6 +2524,11 @@
 				"@types/send": "*"
 			}
 		},
+		"node_modules/@types/statuses": {
+			"version": "2.0.5",
+			"resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz",
+			"integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A=="
+		},
 		"node_modules/@types/tough-cookie": {
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz",
@@ -2382,6 +2555,11 @@
 				"@types/webidl-conversions": "*"
 			}
 		},
+		"node_modules/@types/wrap-ansi": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz",
+			"integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g=="
+		},
 		"node_modules/@typescript-eslint/eslint-plugin": {
 			"version": "6.7.4",
 			"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.4.tgz",
@@ -2799,11 +2977,35 @@
 				"url": "https://github.com/sponsors/epoberezkin"
 			}
 		},
+		"node_modules/ansi-escapes": {
+			"version": "4.3.2",
+			"resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz",
+			"integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==",
+			"dependencies": {
+				"type-fest": "^0.21.3"
+			},
+			"engines": {
+				"node": ">=8"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/ansi-escapes/node_modules/type-fest": {
+			"version": "0.21.3",
+			"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz",
+			"integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==",
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
 		"node_modules/ansi-regex": {
 			"version": "5.0.1",
 			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
 			"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-			"dev": true,
 			"engines": {
 				"node": ">=8"
 			}
@@ -2812,7 +3014,6 @@
 			"version": "4.3.0",
 			"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
 			"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-			"dev": true,
 			"dependencies": {
 				"color-convert": "^2.0.1"
 			},
@@ -3195,8 +3396,6 @@
 			"resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz",
 			"integrity": "sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==",
 			"hasInstallScript": true,
-			"optional": true,
-			"peer": true,
 			"dependencies": {
 				"node-gyp-build": "^4.3.0"
 			},
@@ -3317,7 +3516,6 @@
 			"version": "4.1.2",
 			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
 			"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
-			"dev": true,
 			"dependencies": {
 				"ansi-styles": "^4.1.0",
 				"supports-color": "^7.1.0"
@@ -3389,6 +3587,54 @@
 			"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
 			"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
 		},
+		"node_modules/cli-spinners": {
+			"version": "2.9.2",
+			"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz",
+			"integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==",
+			"engines": {
+				"node": ">=6"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/cli-width": {
+			"version": "4.1.0",
+			"resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz",
+			"integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==",
+			"engines": {
+				"node": ">= 12"
+			}
+		},
+		"node_modules/cliui": {
+			"version": "8.0.1",
+			"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
+			"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
+			"dependencies": {
+				"string-width": "^4.2.0",
+				"strip-ansi": "^6.0.1",
+				"wrap-ansi": "^7.0.0"
+			},
+			"engines": {
+				"node": ">=12"
+			}
+		},
+		"node_modules/cliui/node_modules/wrap-ansi": {
+			"version": "7.0.0",
+			"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+			"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+			"dependencies": {
+				"ansi-styles": "^4.0.0",
+				"string-width": "^4.1.0",
+				"strip-ansi": "^6.0.0"
+			},
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+			}
+		},
 		"node_modules/code-red": {
 			"version": "1.0.4",
 			"resolved": "https://registry.npmjs.org/code-red/-/code-red-1.0.4.tgz",
@@ -3538,7 +3784,6 @@
 			"version": "0.5.0",
 			"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
 			"integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
-			"dev": true,
 			"engines": {
 				"node": ">= 0.6"
 			}
@@ -4351,6 +4596,14 @@
 				"node": ">=0.8.x"
 			}
 		},
+		"node_modules/eventsource": {
+			"version": "2.0.2",
+			"resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz",
+			"integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==",
+			"engines": {
+				"node": ">=12.0.0"
+			}
+		},
 		"node_modules/execa": {
 			"version": "5.1.1",
 			"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
@@ -4803,6 +5056,14 @@
 				"node": ">=14"
 			}
 		},
+		"node_modules/get-caller-file": {
+			"version": "2.0.5",
+			"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+			"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+			"engines": {
+				"node": "6.* || 8.* || >= 10.*"
+			}
+		},
 		"node_modules/get-func-name": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@@ -4971,6 +5232,14 @@
 			"integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
 			"dev": true
 		},
+		"node_modules/graphql": {
+			"version": "16.8.1",
+			"resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz",
+			"integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==",
+			"engines": {
+				"node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
+			}
+		},
 		"node_modules/gtoken": {
 			"version": "7.1.0",
 			"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
@@ -5013,7 +5282,6 @@
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
 			"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-			"dev": true,
 			"engines": {
 				"node": ">=8"
 			}
@@ -5067,6 +5335,11 @@
 				"node": ">= 0.4"
 			}
 		},
+		"node_modules/headers-polyfill": {
+			"version": "4.0.3",
+			"resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz",
+			"integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ=="
+		},
 		"node_modules/help-me": {
 			"version": "5.0.0",
 			"resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
@@ -5356,6 +5629,14 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/is-fullwidth-code-point": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+			"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/is-glob": {
 			"version": "4.0.3",
 			"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
@@ -5373,6 +5654,11 @@
 			"integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==",
 			"dev": true
 		},
+		"node_modules/is-node-process": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz",
+			"integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw=="
+		},
 		"node_modules/is-number": {
 			"version": "7.0.0",
 			"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
@@ -6101,6 +6387,77 @@
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 			"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 		},
+		"node_modules/msw": {
+			"version": "2.3.0",
+			"resolved": "https://registry.npmjs.org/msw/-/msw-2.3.0.tgz",
+			"integrity": "sha512-cDr1q/QTMzaWhY8n9lpGhceY209k29UZtdTgJ3P8Bzne3TSMchX2EM/ldvn4ATLOktpCefCU2gcEgzHc31GTPw==",
+			"hasInstallScript": true,
+			"dependencies": {
+				"@bundled-es-modules/cookie": "^2.0.0",
+				"@bundled-es-modules/statuses": "^1.0.1",
+				"@inquirer/confirm": "^3.0.0",
+				"@mswjs/cookies": "^1.1.0",
+				"@mswjs/interceptors": "^0.29.0",
+				"@open-draft/until": "^2.1.0",
+				"@types/cookie": "^0.6.0",
+				"@types/statuses": "^2.0.4",
+				"chalk": "^4.1.2",
+				"graphql": "^16.8.1",
+				"headers-polyfill": "^4.0.2",
+				"is-node-process": "^1.2.0",
+				"outvariant": "^1.4.2",
+				"path-to-regexp": "^6.2.0",
+				"strict-event-emitter": "^0.5.1",
+				"type-fest": "^4.9.0",
+				"yargs": "^17.7.2"
+			},
+			"bin": {
+				"msw": "cli/index.js"
+			},
+			"engines": {
+				"node": ">=18"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/mswjs"
+			},
+			"peerDependencies": {
+				"typescript": ">= 4.7.x"
+			},
+			"peerDependenciesMeta": {
+				"typescript": {
+					"optional": true
+				}
+			}
+		},
+		"node_modules/msw/node_modules/@types/cookie": {
+			"version": "0.6.0",
+			"resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz",
+			"integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA=="
+		},
+		"node_modules/msw/node_modules/path-to-regexp": {
+			"version": "6.2.2",
+			"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz",
+			"integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw=="
+		},
+		"node_modules/msw/node_modules/type-fest": {
+			"version": "4.18.2",
+			"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.18.2.tgz",
+			"integrity": "sha512-+suCYpfJLAe4OXS6+PPXjW3urOS4IoP9waSiLuXfLgqZODKw/aWwASvzqE886wA0kQgGy0mIWyhd87VpqIy6Xg==",
+			"engines": {
+				"node": ">=16"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/mute-stream": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz",
+			"integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==",
+			"engines": {
+				"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
+			}
+		},
 		"node_modules/mz": {
 			"version": "2.7.0",
 			"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -6233,8 +6590,6 @@
 			"version": "4.6.1",
 			"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz",
 			"integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==",
-			"optional": true,
-			"peer": true,
 			"bin": {
 				"node-gyp-build": "bin.js",
 				"node-gyp-build-optional": "optional.js",
@@ -6473,6 +6828,11 @@
 				"node": ">= 0.8.0"
 			}
 		},
+		"node_modules/outvariant": {
+			"version": "1.4.2",
+			"resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.2.tgz",
+			"integrity": "sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ=="
+		},
 		"node_modules/p-limit": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -6866,11 +7226,11 @@
 			"integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="
 		},
 		"node_modules/playwright": {
-			"version": "1.43.1",
-			"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.1.tgz",
-			"integrity": "sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==",
+			"version": "1.40.0",
+			"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.40.0.tgz",
+			"integrity": "sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw==",
 			"dependencies": {
-				"playwright-core": "1.43.1"
+				"playwright-core": "1.40.0"
 			},
 			"bin": {
 				"playwright": "cli.js"
@@ -6906,6 +7266,17 @@
 				"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
 			}
 		},
+		"node_modules/playwright/node_modules/playwright-core": {
+			"version": "1.40.0",
+			"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.40.0.tgz",
+			"integrity": "sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q==",
+			"bin": {
+				"playwright-core": "cli.js"
+			},
+			"engines": {
+				"node": ">=16"
+			}
+		},
 		"node_modules/postcss": {
 			"version": "8.4.35",
 			"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.35.tgz",
@@ -7542,6 +7913,14 @@
 				"node": ">= 12.13.0"
 			}
 		},
+		"node_modules/require-directory": {
+			"version": "2.1.1",
+			"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+			"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
 		"node_modules/requires-port": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
@@ -7783,6 +8162,14 @@
 			"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
 			"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
 		},
+		"node_modules/semiver": {
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/semiver/-/semiver-1.1.0.tgz",
+			"integrity": "sha512-QNI2ChmuioGC1/xjyYwyZYADILWyW6AmS1UH6gDj/SFUUUS4MBAWs/7mxnkRPc/F4iHezDP+O8t0dO8WHiEOdg==",
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/semver": {
 			"version": "7.6.2",
 			"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
@@ -7885,9 +8272,9 @@
 			"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
 		},
 		"node_modules/sharp": {
-			"version": "0.33.3",
-			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz",
-			"integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==",
+			"version": "0.33.4",
+			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.4.tgz",
+			"integrity": "sha512-7i/dt5kGl7qR4gwPRD2biwD2/SvBn3O04J77XKFgL2OnZtQw+AG9wnuS/csmu80nPRHLYE9E41fyEiG8nhH6/Q==",
 			"hasInstallScript": true,
 			"dependencies": {
 				"color": "^4.2.3",
@@ -7902,8 +8289,8 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-darwin-arm64": "0.33.3",
-				"@img/sharp-darwin-x64": "0.33.3",
+				"@img/sharp-darwin-arm64": "0.33.4",
+				"@img/sharp-darwin-x64": "0.33.4",
 				"@img/sharp-libvips-darwin-arm64": "1.0.2",
 				"@img/sharp-libvips-darwin-x64": "1.0.2",
 				"@img/sharp-libvips-linux-arm": "1.0.2",
@@ -7912,15 +8299,15 @@
 				"@img/sharp-libvips-linux-x64": "1.0.2",
 				"@img/sharp-libvips-linuxmusl-arm64": "1.0.2",
 				"@img/sharp-libvips-linuxmusl-x64": "1.0.2",
-				"@img/sharp-linux-arm": "0.33.3",
-				"@img/sharp-linux-arm64": "0.33.3",
-				"@img/sharp-linux-s390x": "0.33.3",
-				"@img/sharp-linux-x64": "0.33.3",
-				"@img/sharp-linuxmusl-arm64": "0.33.3",
-				"@img/sharp-linuxmusl-x64": "0.33.3",
-				"@img/sharp-wasm32": "0.33.3",
-				"@img/sharp-win32-ia32": "0.33.3",
-				"@img/sharp-win32-x64": "0.33.3"
+				"@img/sharp-linux-arm": "0.33.4",
+				"@img/sharp-linux-arm64": "0.33.4",
+				"@img/sharp-linux-s390x": "0.33.4",
+				"@img/sharp-linux-x64": "0.33.4",
+				"@img/sharp-linuxmusl-arm64": "0.33.4",
+				"@img/sharp-linuxmusl-x64": "0.33.4",
+				"@img/sharp-wasm32": "0.33.4",
+				"@img/sharp-win32-ia32": "0.33.4",
+				"@img/sharp-win32-x64": "0.33.4"
 			}
 		},
 		"node_modules/shebang-command": {
@@ -8163,6 +8550,11 @@
 				"queue-tick": "^1.0.1"
 			}
 		},
+		"node_modules/strict-event-emitter": {
+			"version": "0.5.1",
+			"resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz",
+			"integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ=="
+		},
 		"node_modules/string_decoder": {
 			"version": "1.3.0",
 			"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -8171,6 +8563,24 @@
 				"safe-buffer": "~5.2.0"
 			}
 		},
+		"node_modules/string-width": {
+			"version": "4.2.3",
+			"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+			"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+			"dependencies": {
+				"emoji-regex": "^8.0.0",
+				"is-fullwidth-code-point": "^3.0.0",
+				"strip-ansi": "^6.0.1"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
+		"node_modules/string-width/node_modules/emoji-regex": {
+			"version": "8.0.0",
+			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+			"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+		},
 		"node_modules/string.prototype.codepointat": {
 			"version": "0.2.1",
 			"resolved": "https://registry.npmjs.org/string.prototype.codepointat/-/string.prototype.codepointat-0.2.1.tgz",
@@ -8180,7 +8590,6 @@
 			"version": "6.0.1",
 			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
 			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-			"dev": true,
 			"dependencies": {
 				"ansi-regex": "^5.0.1"
 			},
@@ -8292,7 +8701,6 @@
 			"version": "7.2.0",
 			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
 			"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-			"dev": true,
 			"dependencies": {
 				"has-flag": "^4.0.0"
 			},
@@ -8952,7 +9360,6 @@
 			"version": "5.2.2",
 			"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz",
 			"integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==",
-			"devOptional": true,
 			"bin": {
 				"tsc": "bin/tsc",
 				"tsserver": "bin/tsserver"
@@ -8995,6 +9402,11 @@
 				"node": ">=14.0"
 			}
 		},
+		"node_modules/undici-types": {
+			"version": "5.26.5",
+			"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+			"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
+		},
 		"node_modules/unicode-trie": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/unicode-trie/-/unicode-trie-2.0.0.tgz",
@@ -9960,6 +10372,19 @@
 			"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
 			"integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q=="
 		},
+		"node_modules/wrap-ansi": {
+			"version": "6.2.0",
+			"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
+			"integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
+			"dependencies": {
+				"ansi-styles": "^4.0.0",
+				"string-width": "^4.1.0",
+				"strip-ansi": "^6.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/wrappy": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -9998,6 +10423,14 @@
 			"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
 			"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
 		},
+		"node_modules/y18n": {
+			"version": "5.0.8",
+			"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+			"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/yallist": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -10012,6 +10445,31 @@
 				"node": ">= 6"
 			}
 		},
+		"node_modules/yargs": {
+			"version": "17.7.2",
+			"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
+			"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
+			"dependencies": {
+				"cliui": "^8.0.1",
+				"escalade": "^3.1.1",
+				"get-caller-file": "^2.0.5",
+				"require-directory": "^2.1.1",
+				"string-width": "^4.2.3",
+				"y18n": "^5.0.5",
+				"yargs-parser": "^21.1.1"
+			},
+			"engines": {
+				"node": ">=12"
+			}
+		},
+		"node_modules/yargs-parser": {
+			"version": "21.1.1",
+			"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+			"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
+			"engines": {
+				"node": ">=12"
+			}
+		},
 		"node_modules/yn": {
 			"version": "3.1.1",
 			"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",

package.json

@@ -54,11 +54,12 @@
 	"type": "module",
 	"dependencies": {
 		"@cliqz/adblocker-playwright": "^1.27.2",
+		"@gradio/client": "^0.19.4",
 		"@huggingface/hub": "^0.5.1",
-		"@huggingface/inference": "^2.6.3",
+		"@huggingface/inference": "^2.7.0",
 		"@iconify-json/bi": "^1.1.21",
-		"@resvg/resvg-js": "^2.6.2",
 		"@playwright/browser-chromium": "^1.43.1",
+		"@resvg/resvg-js": "^2.6.2",
 		"@xenova/transformers": "^2.16.1",
 		"autoprefixer": "^10.4.14",
 		"browser-image-resizer": "^2.4.1",
@@ -87,7 +88,7 @@
 		"satori-html": "^0.3.2",
 		"sbd": "^1.0.19",
 		"serpapi": "^1.1.1",
-		"sharp": "^0.33.3",
+		"sharp": "^0.33.4",
 		"tailwind-scrollbar": "^3.0.0",
 		"tailwindcss": "^3.4.0",
 		"uuid": "^9.0.1",

src/lib/actions/clickOutside.ts

@@ -1,4 +1,4 @@
-export function clickOutside(element: HTMLDialogElement, callbackFunction: () => void) {
+export function clickOutside(element: HTMLElement, callbackFunction: () => void) {
 	function onClick(event: MouseEvent) {
 		if (!element.contains(event.target as Node)) {
 			callbackFunction();

src/lib/buildPrompt.ts

@@ -1,8 +1,11 @@
 import type { EndpointParameters } from "./server/endpoints/endpoints";
 import type { BackendModel } from "./server/models";
+import type { Tool, ToolResult } from "./types/Tool";
 
 type buildPromptOptions = Pick<EndpointParameters, "messages" | "preprompt" | "continueMessage"> & {
 	model: BackendModel;
+	tools?: Tool[];
+	toolResults?: ToolResult[];
 };
 
 export async function buildPrompt({
@@ -10,11 +13,18 @@ export async function buildPrompt({
 	model,
 	preprompt,
 	continueMessage,
+	tools,
+	toolResults,
 }: buildPromptOptions): Promise<string> {
 	const filteredMessages = messages.filter((m) => m.from !== "system");
 
 	let prompt = model
-		.chatPromptRender({ messages: filteredMessages, preprompt })
+		.chatPromptRender({
+			messages: filteredMessages,
+			preprompt,
+			tools,
+			toolResults,
+		})
 		// Not super precise, but it's truncated in the model's backend anyway
 		.split(" ")
 		.slice(-(model.parameters?.truncate ?? 0))

src/lib/components/OpenWebSearchResults.svelte

@@ -1,16 +1,22 @@
 <script lang="ts">
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import {
+		MessageWebSearchUpdateType,
+		type MessageWebSearchUpdate,
+	} from "$lib/types/MessageUpdate";
+	import { isMessageWebSearchSourcesUpdate } from "$lib/utils/messageUpdates";
 
 	import CarbonError from "~icons/carbon/error-filled";
 	import EosIconsLoading from "~icons/eos-icons/loading";
 	import IconInternet from "./icons/IconInternet.svelte";
 
 	export let classNames = "";
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let webSearchMessages: MessageWebSearchUpdate[] = [];
 
-	$: sources = webSearchMessages.find((m) => m.sources)?.sources;
-	$: lastMessage = webSearchMessages.filter((m) => m.messageType !== "sources").slice(-1)[0];
-	$: loading = !sources && lastMessage.messageType !== "error";
+	$: sources = webSearchMessages.find(isMessageWebSearchSourcesUpdate)?.sources;
+	$: lastMessage = webSearchMessages
+		.filter((update) => update.subtype !== MessageWebSearchUpdateType.Sources)
+		.at(-1) as MessageWebSearchUpdate;
+	$: loading = !sources && lastMessage.subtype !== MessageWebSearchUpdateType.Error;
 </script>
 
 <details
@@ -47,7 +53,7 @@
 				{#if sources}
 					Completed
 				{:else}
-					{lastMessage.message}
+					{"message" in lastMessage ? lastMessage.message : "An error occurred"}
 				{/if}
 			</dt>
 		</dl>
@@ -61,7 +67,7 @@
 		{:else}
 			<ol>
 				{#each webSearchMessages as message}
-					{#if message.messageType === "update"}
+					{#if message.subtype === MessageWebSearchUpdateType.Update}
 						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
 							<div class="flex items-start">
 								<div
@@ -79,7 +85,7 @@
 								</p>
 							{/if}
 						</li>
-					{:else if message.messageType === "error"}
+					{:else if message.subtype === MessageWebSearchUpdateType.Error}
 						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
 							<div class="flex items-start">
 								<CarbonError

src/lib/components/ToolsMenu.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { page } from "$app/stores";
+	import { clickOutside } from "$lib/actions/clickOutside";
+	import { useSettingsStore } from "$lib/stores/settings";
+	import type { ToolFront } from "$lib/types/Tool";
+	import { isHuggingChat } from "$lib/utils/isHuggingChat";
+	import IconTool from "./icons/IconTool.svelte";
+	import CarbonInformation from "~icons/carbon/information";
+
+	export let loading = false;
+	const settings = useSettingsStore();
+
+	let detailsEl: HTMLDetailsElement;
+
+	// active tools are all the checked tools, either from settings or on by default
+	$: activeToolCount = $page.data.tools.filter(
+		(tool: ToolFront) => $settings?.tools?.[tool.name] ?? tool.isOnByDefault
+	).length;
+</script>
+
+<details
+	class="group relative bottom-0 h-full min-h-8"
+	bind:this={detailsEl}
+	use:clickOutside={() => {
+		if (detailsEl.hasAttribute("open")) {
+			detailsEl.removeAttribute("open");
+		}
+	}}
+>
+	<summary
+		class="absolute bottom-0 flex h-8
+	cursor-pointer select-none items-center gap-1 rounded-lg border bg-white px-2 py-1.5 shadow-sm hover:shadow-none dark:border-gray-800 dark:bg-gray-900"
+	>
+		<IconTool classNames="dark:text-purple-600" />
+		Tools
+		<span class="text-gray-400 dark:text-gray-500"> ({activeToolCount}) </span>
+	</summary>
+	<div
+		class="absolute bottom-10 h-max w-max select-none items-center gap-1 rounded-lg border bg-white p-0.5 shadow-sm dark:border-gray-800 dark:bg-gray-900"
+	>
+		<div class="grid grid-cols-2 gap-x-6 gap-y-1 p-3">
+			<div class="col-span-2 mb-1 flex items-center gap-1.5 text-sm text-gray-500">
+				Available tools
+				{#if isHuggingChat}
+					<a
+						href="https://huggingface.co/spaces/huggingchat/chat-ui/discussions/454"
+						target="_blank"
+						class="hover:brightness-0 dark:hover:brightness-200"
+						><CarbonInformation class="text-xs" /></a
+					>
+				{/if}
+			</div>
+			{#each $page.data.tools as tool}
+				{@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault}
+				<div class="flex items-center gap-1.5">
+					<input
+						type="checkbox"
+						id={tool.name}
+						checked={isChecked}
+						disabled={loading}
+						on:click={async () => {
+							await settings.instantSet({
+								tools: {
+									...$settings.tools,
+									[tool.name]: !isChecked,
+								},
+							});
+						}}
+					/>
+					<label class="cursor-pointer" for={tool.name}>{tool.displayName ?? tool.name} </label>
+				</div>
+			{/each}
+		</div>
+	</div>
+</details>

src/lib/components/UploadBtn.svelte

@@ -3,21 +3,26 @@
 
 	export let classNames = "";
 	export let files: File[];
-	let filelist: FileList;
 
-	$: if (filelist) {
-		files = Array.from(filelist);
-	}
+	/**
+	 * Due to a bug with Svelte, we cannot use bind:files with multiple
+	 * So we use this workaround
+	 **/
+	const onFileChange = (e: Event) => {
+		if (!e.target) return;
+		const target = e.target as HTMLInputElement;
+		files = [...files, ...(target.files ?? [])];
+	};
 </script>
 
 <button
-	class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
+	class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
 >
 	<input
-		bind:files={filelist}
 		class="absolute w-full cursor-pointer opacity-0"
 		type="file"
-		accept="image/*"
+		on:change={onFileChange}
+		accept="*/*"
 	/>
-	<CarbonUpload class="mr-2 text-xs " /> Upload image
+	<CarbonUpload class="mr-2 text-xxs" /> Upload file
 </button>

src/lib/components/chat/ChatMessage.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { marked, type MarkedOptions } from "marked";
 	import markedKatex from "marked-katex-extension";
-	import type { Message } from "$lib/types/Message";
+	import type { Message, MessageFile } from "$lib/types/Message";
 	import { afterUpdate, createEventDispatcher, tick } from "svelte";
 	import { deepestChild } from "$lib/utils/deepestChild";
 	import { page } from "$app/stores";
@@ -16,14 +16,25 @@
 	import CarbonPen from "~icons/carbon/pen";
 	import CarbonChevronLeft from "~icons/carbon/chevron-left";
 	import CarbonChevronRight from "~icons/carbon/chevron-right";
-
+	import CarbonTools from "~icons/carbon/tools";
 	import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken";
 	import type { Model } from "$lib/types/Model";
+	import UploadedFile from "./UploadedFile.svelte";
 
 	import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import {
+		MessageToolUpdateType,
+		MessageWebSearchUpdateType,
+		type MessageToolUpdate,
+		type MessageWebSearchSourcesUpdate,
+		type MessageWebSearchUpdate,
+	} from "$lib/types/MessageUpdate";
+	import { isMessageToolCallUpdate, isMessageToolResultUpdate } from "$lib/utils/messageUpdates";
+	import type { ToolFront } from "$lib/types/Tool";
 	import { base } from "$app/paths";
 	import { useConvTreeStore } from "$lib/stores/convTree";
+	import Modal from "../Modal.svelte";
+	import { toolHasName } from "$lib/utils/tools";
 
 	function sanitizeMd(md: string) {
 		let ret = md
@@ -58,6 +69,8 @@
 
 	$: message = messages.find((m) => m.id === id) ?? ({} as Message);
 
+	$: urlNotTrailing = $page.url.pathname.replace(/\/$/, "");
+
 	const dispatch = createEventDispatcher<{
 		retry: { content?: string; id: Message["id"] };
 		vote: { score: Message["score"]; id: Message["id"] };
@@ -129,19 +142,33 @@
 	}
 
 	$: searchUpdates = (message.updates?.filter(({ type }) => type === "webSearch") ??
-		[]) as WebSearchUpdate[];
+		[]) as MessageWebSearchUpdate[];
+
+	// filter all updates with type === "tool" then group them by uuid field
+
+	$: toolUpdates = message.updates
+		?.filter(({ type }) => type === "tool")
+		.reduce((acc, update) => {
+			if (update.type !== "tool") {
+				return acc;
+			}
+			acc[update.uuid] = acc[update.uuid] ?? [];
+			acc[update.uuid].push(update);
+			return acc;
+		}, {} as Record<string, MessageToolUpdate[]>);
 
-	$: downloadLink =
-		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
+	$: downloadLink = urlNotTrailing + `/message/${message.id}/prompt`;
 
 	let webSearchIsDone = true;
 
-	$: webSearchIsDone =
-		searchUpdates.length > 0 && searchUpdates[searchUpdates.length - 1].messageType === "sources";
+	$: webSearchIsDone = searchUpdates.some(
+		(update) => update.subtype === MessageWebSearchUpdateType.Finished
+	);
 
-	$: webSearchSources =
-		searchUpdates &&
-		searchUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources;
+	$: webSearchSources = searchUpdates?.find(
+		(update): update is MessageWebSearchSourcesUpdate =>
+			update.subtype === MessageWebSearchUpdateType.Sources
+	)?.sources;
 
 	$: if (isCopied) {
 		setTimeout(() => {
@@ -177,8 +204,32 @@
 	const convTreeStore = useConvTreeStore();
 
 	$: if (message.children?.length === 0) $convTreeStore.leaf = message.id;
+
+	$: modalImageToShow = null as MessageFile | null;
+
+	const availableTools: ToolFront[] = $page.data.tools;
 </script>
 
+{#if modalImageToShow}
+	<!-- show the image file full screen, click outside to exit -->
+	<Modal width="sm:max-w-[500px]" on:close={() => (modalImageToShow = null)}>
+		{#if modalImageToShow.type === "hash"}
+			<img
+				src={urlNotTrailing + "/output/" + modalImageToShow.value}
+				alt="input from user"
+				class="aspect-auto"
+			/>
+		{:else}
+			<!-- handle the case where this is a base64 encoded image -->
+			<img
+				src={`data:${modalImageToShow.mime};base64,${modalImageToShow.value}`}
+				alt="input from user"
+				class="aspect-auto"
+			/>
+		{/if}
+	</Modal>
+{/if}
+
 {#if message.from === "assistant"}
 	<div
 		class="group relative -mb-6 flex items-start justify-start gap-4 pb-4 leading-relaxed"
@@ -202,6 +253,29 @@
 		<div
 			class="relative min-h-[calc(2rem+theme(spacing[3.5])*2)] min-w-[60px] break-words rounded-2xl border border-gray-100 bg-gradient-to-br from-gray-50 px-5 py-3.5 text-gray-600 prose-pre:my-2 dark:border-gray-800 dark:from-gray-800/40 dark:text-gray-300"
 		>
+			{#if message.files?.length}
+				<div class="flex h-fit flex-wrap gap-x-5 gap-y-2">
+					{#each message.files as file}
+						<!-- handle the case where this is a hash that points to an image in the db, hash is always 64 char long -->
+						<button on:click={() => (modalImageToShow = file)}>
+							{#if file.type === "hash"}
+								<img
+									src={urlNotTrailing + "/output/" + file.value}
+									alt="output from assistant"
+									class="my-2 aspect-auto max-h-48 cursor-pointer rounded-lg shadow-lg"
+								/>
+							{:else}
+								<!-- handle the case where this is a base64 encoded image -->
+								<img
+									src={`data:${file.mime};base64,${file.value}`}
+									alt="output from assistant"
+									class="my-2 aspect-auto max-h-48 cursor-pointer rounded-lg shadow-lg"
+								/>
+							{/if}
+						</button>
+					{/each}
+				</div>
+			{/if}
 			{#if searchUpdates && searchUpdates.length > 0}
 				<OpenWebSearchResults
 					classNames={tokens.length ? "mb-3.5" : ""}
@@ -209,6 +283,72 @@
 				/>
 			{/if}
 
+			{#if toolUpdates}
+				{#each Object.values(toolUpdates) as tool}
+					{#if tool.length}
+						{@const toolName = tool.find(isMessageToolCallUpdate)?.call.name}
+						{@const toolDone = tool.some(isMessageToolResultUpdate)}
+						{#if toolName && toolName !== "websearch"}
+							<details
+								class="group/tool my-2.5 w-fit cursor-pointer rounded-lg border border-gray-200 bg-white pl-1 pr-2.5 text-sm shadow-sm transition-all open:mb-3
+								open:border-purple-500/10 open:bg-purple-600/5 open:shadow-sm dark:border-gray-800 dark:bg-gray-900 open:dark:border-purple-800/40 open:dark:bg-purple-800/10"
+							>
+								<summary
+									class="flex select-none list-none items-center gap-1.5 py-1 group-open/tool:text-purple-700 group-open/tool:dark:text-purple-300"
+								>
+									<div
+										class="relative grid size-[22px] place-items-center rounded bg-purple-600/10 dark:bg-purple-600/20"
+									>
+										<svg
+											class="absolute inset-0 text-purple-500/40 transition-opacity"
+											class:invisible={toolDone}
+											width="22"
+											height="22"
+											viewBox="0 0 38 38"
+											fill="none"
+											xmlns="http://www.w3.org/2000/svg"
+										>
+											<path
+												class="loading-path"
+												d="M8 2.5H30C30 2.5 35.5 2.5 35.5 8V30C35.5 30 35.5 35.5 30 35.5H8C8 35.5 2.5 35.5 2.5 30V8C2.5 8 2.5 2.5 8 2.5Z"
+												stroke="currentColor"
+												stroke-width="1"
+												stroke-linecap="round"
+												id="shape"
+											/>
+										</svg>
+										<CarbonTools class="text-xs text-purple-700 dark:text-purple-500" />
+									</div>
+
+									<span>
+										{toolDone ? "Called" : "Calling"} tool
+										<span class="font-semibold"
+											>{availableTools.find((el) => toolHasName(toolName, el))?.displayName}</span
+										>
+									</span>
+								</summary>
+								{#each tool as toolUpdate}
+									{#if toolUpdate.subtype === MessageToolUpdateType.Call}
+										<div class="mt-1 flex items-center gap-2 opacity-80">
+											<h3 class="text-sm">Parameters</h3>
+											<div class="h-px flex-1 bg-gradient-to-r from-gray-500/20" />
+										</div>
+										<ul class="py-1 text-sm">
+											{#each Object.entries(toolUpdate.call.parameters ?? {}) as [k, v]}
+												<li>
+													<span class="font-semibold">{k}</span>:
+													<span>{v}</span>
+												</li>
+											{/each}
+										</ul>
+									{/if}
+								{/each}
+							</details>
+						{/if}
+					{/if}
+				{/each}
+			{/if}
+
 			<div
 				class="prose max-w-none max-sm:prose-sm dark:prose-invert prose-headings:font-semibold prose-h1:text-lg prose-h2:text-base prose-h3:text-base prose-pre:bg-gray-800 dark:prose-pre:bg-gray-900"
 				bind:this={contentEl}
@@ -244,7 +384,7 @@
 				</div>
 			{/if}
 		</div>
-		{#if !loading && message.content}
+		{#if !loading && (message.content || toolUpdates)}
 			<div
 				class="absolute bottom-1 right-0 -mb-4 flex max-md:transition-all md:bottom-0 md:group-hover:visible md:group-hover:opacity-100
 		{message.score ? 'visible opacity-100' : 'invisible max-md:-translate-y-4 max-md:opacity-0'}
@@ -304,24 +444,16 @@
 		on:click={() => (isTapped = !isTapped)}
 		on:keydown={() => (isTapped = !isTapped)}
 	>
-		<div class="flex w-full flex-col">
-			{#if message.files && message.files.length > 0}
-				<div class="mx-auto grid w-fit grid-cols-2 gap-5 px-5">
+		<div class="flex w-full flex-col gap-2">
+			{#if message.files?.length}
+				<div class="flex w-fit gap-4 px-5">
 					{#each message.files as file}
-						<!-- handle the case where this is a hash that points to an image in the db -->
-						{#if file.type === "hash"}
-							<img
-								src={$page.url.pathname + "/output/" + file.value}
-								alt="input from user"
-								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
-							/>
+						{#if file.mime.startsWith("image/")}
+							<button on:click={() => (modalImageToShow = file)}>
+								<UploadedFile {file} canClose={false} />
+							</button>
 						{:else}
-							<!-- handle the case where this is a base64 encoded image -->
-							<img
-								src={`data:${file.mime};base64,${file.value}`}
-								alt="input from user"
-								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
-							/>
+							<UploadedFile {file} canClose={false} />
 						{/if}
 					{/each}
 				</div>
@@ -458,3 +590,20 @@
 		</svelte:fragment>
 	</svelte:self>
 {/if}
+
+<style>
+	details summary::-webkit-details-marker {
+		display: none;
+	}
+
+	.loading-path {
+		stroke-dasharray: 61.45;
+		animation: loading 2s linear infinite;
+	}
+
+	@keyframes loading {
+		to {
+			stroke-dashoffset: 122.9;
+		}
+	}
+</style>

src/lib/components/chat/ChatWindow.svelte

@@ -1,11 +1,10 @@
 <script lang="ts">
-	import type { Message } from "$lib/types/Message";
+	import type { Message, MessageFile } from "$lib/types/Message";
 	import { createEventDispatcher, onDestroy, tick } from "svelte";
 
 	import CarbonSendAltFilled from "~icons/carbon/send-alt-filled";
 	import CarbonExport from "~icons/carbon/export";
 	import CarbonStopFilledAlt from "~icons/carbon/stop-filled-alt";
-	import CarbonClose from "~icons/carbon/close";
 	import CarbonCheckmark from "~icons/carbon/checkmark";
 	import CarbonCaretDown from "~icons/carbon/caret-down";
 
@@ -15,6 +14,7 @@
 	import StopGeneratingBtn from "../StopGeneratingBtn.svelte";
 	import type { Model } from "$lib/types/Model";
 	import WebSearchToggle from "../WebSearchToggle.svelte";
+	import ToolsMenu from "../ToolsMenu.svelte";
 	import LoginModal from "../LoginModal.svelte";
 	import { page } from "$app/stores";
 	import FileDropzone from "./FileDropzone.svelte";
@@ -32,6 +32,7 @@
 	import SystemPromptModal from "../SystemPromptModal.svelte";
 	import ChatIntroduction from "./ChatIntroduction.svelte";
 	import { useConvTreeStore } from "$lib/stores/convTree";
+	import UploadedFile from "./UploadedFile.svelte";
 
 	export let messages: Message[] = [];
 	export let loading = false;
@@ -92,8 +93,8 @@
 		(lastMessage.from === "user" ||
 			lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1);
 
-	$: sources = files?.map((file) =>
-		file2base64(file).then((value) => ({ type: "base64", value, mime: file.type }))
+	$: sources = files?.map<Promise<MessageFile>>((file) =>
+		file2base64(file).then((value) => ({ type: "base64", value, mime: file.type, name: file.name }))
 	);
 
 	function onShare() {
@@ -139,7 +140,9 @@
 		use:snapScrollToBottom={messages.length ? [...messages] : false}
 		bind:this={chatContainer}
 	>
-		<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
+		<div
+			class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl xl:pt-10"
+		>
 			{#if $page.data?.assistant && !!messages.length}
 				<a
 					class="mx-auto flex items-center gap-1.5 rounded-full border border-gray-100 bg-gray-50 py-1 pl-1 pr-3 text-sm text-gray-800 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-200 dark:hover:bg-gray-700"
@@ -167,7 +170,7 @@
 			{/if}
 
 			{#if messages.length > 0}
-				<div class="flex h-max flex-col gap-6 pb-52">
+				<div class="flex h-max flex-col gap-6 pb-52 2xl:gap-7">
 					<ChatMessage
 						{loading}
 						{messages}
@@ -235,22 +238,12 @@
 			<div class="flex flex-row flex-wrap justify-center gap-2.5 max-md:pb-3">
 				{#each sources as source, index}
 					{#await source then src}
-						<div class="relative h-16 w-16 overflow-hidden rounded-lg shadow-lg">
-							<img
-								src={`data:${src.mime};base64,${src.value}`}
-								alt="input content"
-								class="h-full w-full rounded-lg bg-gray-400 object-cover dark:bg-gray-900"
-							/>
-							<!-- add a button on top that deletes this image from sources -->
-							<button
-								class="absolute left-1 top-1"
-								on:click={() => {
-									files = files.filter((_, i) => i !== index);
-								}}
-							>
-								<CarbonClose class="text-md font-black text-gray-300  hover:text-gray-100" />
-							</button>
-						</div>
+						<UploadedFile
+							file={src}
+							on:close={() => {
+								files = files.filter((_, i) => i !== index);
+							}}
+						/>
 					{/await}
 				{/each}
 			</div>
@@ -258,8 +251,12 @@
 
 		<div class="w-full">
 			<div class="flex w-full pb-3">
-				{#if $page.data.settings?.searchEnabled && !assistant}
-					<WebSearchToggle />
+				{#if !assistant}
+					{#if currentModel.tools}
+						<ToolsMenu {loading} />
+					{:else if $page.data.settings?.searchEnabled}
+						<WebSearchToggle />
+					{/if}
 				{/if}
 				{#if loading}
 					<StopGeneratingBtn classNames="ml-auto" on:click={() => dispatch("stop")} />
@@ -276,7 +273,7 @@
 					/>
 				{:else}
 					<div class="ml-auto gap-2">
-						{#if currentModel.multimodal}
+						{#if currentModel.multimodal || currentModel.tools}
 							<UploadBtn bind:files classNames="ml-auto" />
 						{/if}
 						{#if messages && lastMessage && lastMessage.interrupted && !isReadOnly}

src/lib/components/chat/UploadedFile.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+	import { createEventDispatcher } from "svelte";
+	import { page } from "$app/stores";
+	import type { MessageFile } from "$lib/types/Message";
+	import CarbonClose from "~icons/carbon/close";
+	import CarbonDocumentBlank from "~icons/carbon/document-blank";
+
+	export let file: MessageFile;
+	export let canClose = true;
+	const dispatch = createEventDispatcher<{ close: void }>();
+</script>
+
+<div
+	class="group relative flex items-center rounded-xl shadow-sm"
+	class:w-24={file.mime.startsWith("image/")}
+	class:w-72={!file.mime.startsWith("image/")}
+>
+	{#if file.mime.startsWith("image/")}
+		<div class="size-24 overflow-hidden rounded-xl">
+			<img
+				src={file.type === "base64"
+					? `data:${file.mime};base64,${file.value}`
+					: $page.url.pathname + "/output/" + file.value}
+				alt={file.name}
+				class="h-full w-full bg-gray-200 object-cover dark:bg-gray-800"
+			/>
+		</div>
+	{:else}
+		<div
+			class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
+		>
+			<div
+				class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
+			>
+				<CarbonDocumentBlank class="text-base text-gray-700 dark:text-gray-300" />
+			</div>
+			<dl class="flex flex-col truncate leading-tight">
+				<dd class="text-sm">
+					{file.name}
+				</dd>
+				<dt class="text-xs text-gray-400">{file.mime.split("/")[1].toUpperCase()}</dt>
+			</dl>
+		</div>
+	{/if}
+	<!-- add a button on top that removes the image -->
+	{#if canClose}
+		<button
+			class="invisible absolute -right-2 -top-2 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
+			on:click={() => dispatch("close")}
+		>
+			<CarbonClose class=" text-xs  text-white" />
+		</button>
+	{/if}
+</div>

src/lib/components/icons/IconTool.svelte

@@ -0,0 +1,16 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	class={classNames}
+	width="1em"
+	height="1em"
+	fill="none"
+	viewBox="0 0 15 15"
+	><path
+		fill="currentColor"
+		d="M8.33 6.56c0 .11-.04.22-.1.31a.55.55 0 0 1-.26.2l-.71.23c-.2.07-.38.18-.53.33-.15.15-.26.33-.33.53l-.24.7a.55.55 0 0 1-.83.26.55.55 0 0 1-.19-.27l-.23-.7a1.37 1.37 0 0 0-.85-.85l-.7-.24a.52.52 0 0 1-.27-.2.56.56 0 0 1 .27-.82l.7-.23a1.4 1.4 0 0 0 .86-.86l.23-.7c.03-.1.1-.19.18-.26a.55.55 0 0 1 .63-.02c.1.06.17.15.2.26l.24.72a1.4 1.4 0 0 0 .86.86l.7.24c.11.04.2.1.26.2.07.09.1.2.1.31ZM13.94 10.14c0 .12-.03.24-.1.34a.55.55 0 0 1-.3.21l-.96.32a2.01 2.01 0 0 0-1.27 1.27l-.33.96a.55.55 0 0 1-.2.28c-.1.07-.23.11-.35.11a.56.56 0 0 1-.57-.4l-.32-.96c-.1-.3-.26-.57-.48-.79-.22-.21-.49-.38-.78-.48l-.97-.32a.62.62 0 0 1-.28-.2.61.61 0 0 1-.02-.7c.07-.1.18-.18.3-.22l.96-.32a1.99 1.99 0 0 0 1.29-1.28l.32-.95a.56.56 0 0 1 .53-.4c.12 0 .24.02.34.09.1.07.18.17.23.28l.32.98a1.99 1.99 0 0 0 1.29 1.28l.95.34c.12.04.22.11.29.21.07.1.11.22.11.35ZM11.48 3.84c.06-.09.1-.19.1-.29a.5.5 0 0 0-.1-.29.55.55 0 0 0-.23-.16l-.35-.12a.55.55 0 0 1-.18-.11.55.55 0 0 1-.11-.19l-.12-.36a.54.54 0 0 0-.18-.23A.55.55 0 0 0 10 2c-.1.01-.2.05-.27.1a.55.55 0 0 0-.16.23l-.12.35a.55.55 0 0 1-.11.19.55.55 0 0 1-.19.1l-.34.12a.5.5 0 0 0-.24.17.55.55 0 0 0-.1.29c0 .1.04.2.1.28.06.08.14.14.23.18l.35.11c.07.02.13.06.18.11.05.05.1.11.11.18l.12.35c.04.1.1.19.18.25.09.05.18.08.28.08.1 0 .2-.03.29-.1a.55.55 0 0 0 .16-.22l.13-.35a.47.47 0 0 1 .29-.3l.35-.11c.1-.03.17-.1.23-.17Z"
+	/></svg
+>

src/lib/migrations/routines/03-add-tools-in-settings.ts

@@ -0,0 +1,29 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+import { logger } from "$lib/server/logger";
+
+const addToolsToSettings: Migration = {
+	_id: new ObjectId("5c9c4c4c4c4c4c4c4c4c4c4c"),
+	name: "Add empty 'tools' record in settings",
+	up: async () => {
+		const { settings } = collections;
+
+		// Find all assistants whose modelId is not in modelIds, and update it to use defaultModelId
+		await settings.updateMany(
+			{
+				tools: { $exists: false },
+			},
+			{ $set: { tools: {} } }
+		);
+
+		settings
+			.createIndex({ tools: 1 })
+			.catch((e) => logger.error("Error creating index during tools migration", e));
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default addToolsToSettings;

src/lib/migrations/routines/04-update-message-updates.ts

@@ -0,0 +1,190 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId, type WithId } from "mongodb";
+import type { Conversation } from "$lib/types/Conversation";
+import type { WebSearchSource } from "$lib/types/WebSearch";
+import {
+	MessageUpdateStatus,
+	MessageUpdateType,
+	MessageWebSearchUpdateType,
+	type MessageUpdate,
+	type MessageWebSearchFinishedUpdate,
+} from "$lib/types/MessageUpdate";
+import type { Message } from "$lib/types/Message";
+import { isMessageWebSearchSourcesUpdate } from "$lib/utils/messageUpdates";
+
+// -----------
+// Copy of the previous message update types
+export type FinalAnswer = {
+	type: "finalAnswer";
+	text: string;
+};
+
+export type TextStreamUpdate = {
+	type: "stream";
+	token: string;
+};
+
+type WebSearchUpdate = {
+	type: "webSearch";
+	messageType: "update" | "error" | "sources";
+	message: string;
+	args?: string[];
+	sources?: WebSearchSource[];
+};
+
+type StatusUpdate = {
+	type: "status";
+	status: "started" | "pending" | "finished" | "error" | "title";
+	message?: string;
+};
+
+type ErrorUpdate = {
+	type: "error";
+	message: string;
+	name: string;
+};
+
+type FileUpdate = {
+	type: "file";
+	sha: string;
+};
+
+type OldMessageUpdate =
+	| FinalAnswer
+	| TextStreamUpdate
+	| WebSearchUpdate
+	| StatusUpdate
+	| ErrorUpdate
+	| FileUpdate;
+
+/** Converts the old message update to the new schema */
+function convertMessageUpdate(message: Message, update: OldMessageUpdate): MessageUpdate | null {
+	try {
+		// Text and files
+		if (update.type === "finalAnswer") {
+			return {
+				type: MessageUpdateType.FinalAnswer,
+				text: update.text,
+				interrupted: message.interrupted ?? false,
+			};
+		} else if (update.type === "stream") {
+			return {
+				type: MessageUpdateType.Stream,
+				token: update.token,
+			};
+		} else if (update.type === "file") {
+			return {
+				type: MessageUpdateType.File,
+				name: "Unknown",
+				sha: update.sha,
+				// assume jpeg but could be any image. should be harmless
+				mime: "image/jpeg",
+			};
+		}
+
+		// Status
+		else if (update.type === "status") {
+			if (update.status === "title") {
+				return {
+					type: MessageUpdateType.Title,
+					title: update.message ?? "New Chat",
+				};
+			}
+			if (update.status === "pending") return null;
+
+			const status =
+				update.status === "started"
+					? MessageUpdateStatus.Started
+					: update.status === "finished"
+					? MessageUpdateStatus.Finished
+					: MessageUpdateStatus.Error;
+			return {
+				type: MessageUpdateType.Status,
+				status,
+				message: update.message,
+			};
+		} else if (update.type === "error") {
+			// Treat it as an error status update
+			return {
+				type: MessageUpdateType.Status,
+				status: MessageUpdateStatus.Error,
+				message: update.message,
+			};
+		}
+
+		// Web Search
+		else if (update.type === "webSearch") {
+			if (update.messageType === "update") {
+				return {
+					type: MessageUpdateType.WebSearch,
+					subtype: MessageWebSearchUpdateType.Update,
+					message: update.message,
+					args: update.args,
+				};
+			} else if (update.messageType === "error") {
+				return {
+					type: MessageUpdateType.WebSearch,
+					subtype: MessageWebSearchUpdateType.Error,
+					message: update.message,
+					args: update.args,
+				};
+			} else if (update.messageType === "sources") {
+				return {
+					type: MessageUpdateType.WebSearch,
+					subtype: MessageWebSearchUpdateType.Sources,
+					message: update.message,
+					sources: update.sources ?? [],
+				};
+			}
+		}
+		console.warn("Unknown message update during migration:", update);
+		return null;
+	} catch (error) {
+		console.error("Error converting message update during migration. Skipping it... Error:", error);
+		return null;
+	}
+}
+
+const updateMessageUpdates: Migration = {
+	_id: new ObjectId("5f9f4f4f4f4f4f4f4f4f4f4f"),
+	name: "Convert message updates to the new schema",
+	up: async () => {
+		const allConversations = collections.conversations.find({}, { projection: { messages: 1 } });
+
+		let conversation: WithId<Pick<Conversation, "messages">> | null = null;
+		while ((conversation = await allConversations.tryNext())) {
+			const messages = conversation.messages.map((message) => {
+				// Convert all of the existing updates to the new schema
+				const updates = message.updates
+					?.map((update) => convertMessageUpdate(message, update as OldMessageUpdate))
+					.filter((update): update is MessageUpdate => Boolean(update));
+
+				// Add the new web search finished update if the sources update exists and webSearch is defined
+				const webSearchSourcesUpdateIndex = updates?.findIndex(isMessageWebSearchSourcesUpdate);
+				if (
+					message.webSearch &&
+					updates &&
+					webSearchSourcesUpdateIndex &&
+					webSearchSourcesUpdateIndex !== -1
+				) {
+					const webSearchFinishedUpdate: MessageWebSearchFinishedUpdate = {
+						type: MessageUpdateType.WebSearch,
+						subtype: MessageWebSearchUpdateType.Finished,
+						webSearch: message.webSearch,
+					};
+					updates.splice(webSearchSourcesUpdateIndex + 1, 0, webSearchFinishedUpdate);
+				}
+				return { ...message, updates };
+			});
+
+			// Set the new messages array
+			await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } });
+		}
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default updateMessageUpdates;

src/lib/migrations/routines/05-update-message-files.ts

@@ -0,0 +1,56 @@
+import { ObjectId, type WithId } from "mongodb";
+import { collections } from "$lib/server/database";
+
+import type { Migration } from ".";
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageFile } from "$lib/types/Message";
+
+const updateMessageFiles: Migration = {
+	_id: new ObjectId("5f9f5f5f5f5f5f5f5f5f5f5f"),
+	name: "Convert message files to the new schema",
+	up: async () => {
+		const allConversations = collections.conversations.find({}, { projection: { messages: 1 } });
+
+		let conversation: WithId<Pick<Conversation, "messages">> | null = null;
+		while ((conversation = await allConversations.tryNext())) {
+			const messages = conversation.messages.map((message) => {
+				const files = (message.files as string[] | undefined)?.map<MessageFile>((file) => {
+					// File is already in the new format
+					if (typeof file !== "string") return file;
+
+					// File was a hash pointing to a file in the bucket
+					if (file.length === 64) {
+						return {
+							type: "hash",
+							name: "unknown.jpg",
+							value: file,
+							mime: "image/jpeg",
+						};
+					}
+					// File was a base64 string
+					else {
+						return {
+							type: "base64",
+							name: "unknown.jpg",
+							value: file,
+							mime: "image/jpeg",
+						};
+					}
+				});
+
+				return {
+					...message,
+					files,
+				};
+			});
+
+			// Set the new messages array
+			await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } });
+		}
+
+		return true;
+	},
+	runEveryTime: false,
+};
+
+export default updateMessageFiles;

src/lib/migrations/routines/index.ts

@@ -3,6 +3,9 @@ import type { ObjectId } from "mongodb";
 import updateSearchAssistant from "./01-update-search-assistants";
 import updateAssistantsModels from "./02-update-assistants-models";
 import type { Database } from "$lib/server/database";
+import addToolsToSettings from "./03-add-tools-in-settings";
+import updateMessageUpdates from "./04-update-message-updates";
+import updateMessageFiles from "./05-update-message-files";
 
 export interface Migration {
 	_id: ObjectId;
@@ -14,4 +17,10 @@ export interface Migration {
 	runEveryTime?: boolean;
 }
 
-export const migrations: Migration[] = [updateSearchAssistant, updateAssistantsModels];
+export const migrations: Migration[] = [
+	updateSearchAssistant,
+	updateAssistantsModels,
+	addToolsToSettings,
+	updateMessageUpdates,
+	updateMessageFiles,
+];

src/lib/server/endpoints/cohere/endpointCohere.ts

@@ -4,6 +4,9 @@ import type { Endpoint } from "../endpoints";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
 import type { Cohere, CohereClient } from "cohere-ai";
 import { buildPrompt } from "$lib/buildPrompt";
+import { ToolResultStatus, type ToolCall } from "$lib/types/Tool";
+import { pipeline, Writable, Readable } from "node:stream";
+import { toolHasName } from "$lib/utils/tools";
 
 export const endpointCohereParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -28,12 +31,18 @@ export async function endpointCohere(
 		throw new Error("Failed to import cohere-ai", { cause: e });
 	}
 
-	return async ({ messages, preprompt, generateSettings, continueMessage }) => {
+	return async ({ messages, preprompt, generateSettings, continueMessage, tools, toolResults }) => {
 		let system = preprompt;
 		if (messages?.[0]?.from === "system") {
 			system = messages[0].content;
 		}
 
+		// Tools must use [A-z_] for their names and directly_answer is banned
+		// It's safe to convert the tool names because we treat - and _ the same
+		tools = tools
+			?.filter((tool) => !toolHasName("directly_answer", tool))
+			.map((tool) => ({ ...tool, name: tool.name.replaceAll("-", "_") }));
+
 		const parameters = { ...model.parameters, ...generateSettings };
 
 		return (async function* () {
@@ -42,10 +51,12 @@ export async function endpointCohere(
 
 			if (raw) {
 				const prompt = await buildPrompt({
-					messages: messages.filter((message) => message.from !== "system"),
+					messages,
 					model,
 					preprompt: system,
 					continueMessage,
+					tools,
+					toolResults,
 				});
 
 				stream = await cohere.chatStream({
@@ -67,18 +78,35 @@ export async function endpointCohere(
 						message: message.content,
 					})) satisfies Cohere.ChatMessage[];
 
-				stream = await cohere.chatStream({
-					model: model.id ?? model.name,
-					chatHistory: formattedMessages.slice(0, -1),
-					message: formattedMessages[formattedMessages.length - 1].message,
-					preamble: system,
-					p: parameters?.top_p,
-					k: parameters?.top_k,
-					maxTokens: parameters?.max_new_tokens,
-					temperature: parameters?.temperature,
-					stopSequences: parameters?.stop,
-					frequencyPenalty: parameters?.frequency_penalty,
-				});
+				stream = await cohere
+					.chatStream({
+						model: model.id ?? model.name,
+						chatHistory: formattedMessages.slice(0, -1),
+						message: formattedMessages[formattedMessages.length - 1].message,
+						preamble: system,
+						p: parameters?.top_p,
+						k: parameters?.top_k,
+						maxTokens: parameters?.max_new_tokens,
+						temperature: parameters?.temperature,
+						stopSequences: parameters?.stop,
+						frequencyPenalty: parameters?.frequency_penalty,
+						tools,
+						toolResults: toolResults?.map((toolResult) => {
+							if (toolResult.status === ToolResultStatus.Error) {
+								return { call: toolResult.call, outputs: [{ error: toolResult.message }] };
+							}
+							return { call: toolResult.call, outputs: toolResult.outputs };
+						}),
+					})
+					.catch(async (err) => {
+						if (!err.body) throw err;
+
+						// Decode the error message and throw
+						const message = await convertStreamToBuffer(err.body).catch(() => {
+							throw err;
+						});
+						throw Error(message, { cause: err });
+					});
 			}
 
 			for await (const output of stream) {
@@ -93,6 +121,18 @@ export async function endpointCohere(
 						generated_text: null,
 						details: null,
 					} satisfies TextGenerationStreamOutput;
+				} else if (output.eventType === "tool-calls-generation") {
+					yield {
+						token: {
+							id: tokenId++,
+							text: "",
+							logprob: 0,
+							special: true,
+							toolCalls: output.toolCalls as ToolCall[],
+						},
+						generated_text: null,
+						details: null,
+					};
 				} else if (output.eventType === "stream-end") {
 					if (["ERROR", "ERROR_TOXIC", "ERROR_LIMIT"].includes(output.finishReason)) {
 						throw new Error(output.finishReason);
@@ -112,3 +152,26 @@ export async function endpointCohere(
 		})();
 	};
 }
+
+async function convertStreamToBuffer(webReadableStream: Readable) {
+	return new Promise<string>((resolve, reject) => {
+		const chunks: Buffer[] = [];
+
+		pipeline(
+			webReadableStream,
+			new Writable({
+				write(chunk, _, callback) {
+					chunks.push(chunk);
+					callback();
+				},
+			}),
+			(err) => {
+				if (err) {
+					reject(err);
+				} else {
+					resolve(Buffer.concat(chunks).toString("utf-8"));
+				}
+			}
+		);
+	});
+}

src/lib/server/endpoints/endpoints.ts

@@ -1,6 +1,6 @@
 import type { Conversation } from "$lib/types/Conversation";
 import type { Message } from "$lib/types/Message";
-import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import type { TextGenerationStreamOutput, TextGenerationStreamToken } from "@huggingface/inference";
 import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi";
 import { z } from "zod";
 import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws";
@@ -26,23 +26,31 @@ import endpointLangserve, {
 	endpointLangserveParametersSchema,
 } from "./langserve/endpointLangserve";
 
+import type { Tool, ToolCall, ToolResult } from "$lib/types/Tool";
+
 export type EndpointMessage = Omit<Message, "id">;
+
 // parameters passed when generating text
 export interface EndpointParameters {
 	messages: EndpointMessage[];
 	preprompt?: Conversation["preprompt"];
 	continueMessage?: boolean; // used to signal that the last message will be extended
 	generateSettings?: Partial<Model["parameters"]>;
+	tools?: Tool[];
+	toolResults?: ToolResult[];
 	isMultimodal?: boolean;
 }
 
 interface CommonEndpoint {
 	weight: number;
 }
+type TextGenerationStreamOutputWithTools = TextGenerationStreamOutput & {
+	token: TextGenerationStreamToken & { toolCalls?: ToolCall[] };
+};
 // type signature for the endpoint
 export type Endpoint = (
 	params: EndpointParameters
-) => Promise<AsyncGenerator<TextGenerationStreamOutput, void, void>>;
+) => Promise<AsyncGenerator<TextGenerationStreamOutputWithTools, void, void>>;
 
 // generator function that takes in parameters for defining the endpoint and return the endpoint
 export type EndpointGenerator<T extends CommonEndpoint> = (parameters: T) => Endpoint;

src/lib/server/endpoints/tgi/endpointTgi.ts

@@ -35,7 +35,15 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 		endpointTgiParametersSchema.parse(input);
 	const imageProcessor = makeImageProcessor(multimodal.image);
 
-	return async ({ messages, preprompt, continueMessage, generateSettings, isMultimodal }) => {
+	return async ({
+		messages,
+		preprompt,
+		continueMessage,
+		generateSettings,
+		tools,
+		toolResults,
+		isMultimodal,
+	}) => {
 		const messagesWithResizedFiles = await Promise.all(
 			messages.map((message) => prepareMessage(Boolean(isMultimodal), message, imageProcessor))
 		);
@@ -45,6 +53,8 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 			preprompt,
 			model,
 			continueMessage,
+			tools,
+			toolResults,
 		});
 
 		return textGenerationStream(

src/lib/server/files/downloadFile.ts

@@ -9,29 +9,26 @@ export async function downloadFile(
 	convId: Conversation["_id"] | SharedConversation["_id"]
 ): Promise<MessageFile & { type: "base64" }> {
 	const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` });
-	let mime = "";
 
-	const buffer = await fileId.next().then(async (file) => {
-		if (!file) {
-			throw error(404, "File not found");
-		}
-		if (file.metadata?.conversation !== convId.toString()) {
-			throw error(403, "You don't have access to this file.");
-		}
+	const file = await fileId.next();
+	if (!file) {
+		throw error(404, "File not found");
+	}
+	if (file.metadata?.conversation !== convId.toString()) {
+		throw error(403, "You don't have access to this file.");
+	}
 
-		mime = file.metadata?.mime;
+	const mime = file.metadata?.mime;
+	const name = file.filename;
 
-		const fileStream = collections.bucket.openDownloadStream(file._id);
+	const fileStream = collections.bucket.openDownloadStream(file._id);
 
-		const fileBuffer = await new Promise<Buffer>((resolve, reject) => {
-			const chunks: Uint8Array[] = [];
-			fileStream.on("data", (chunk) => chunks.push(chunk));
-			fileStream.on("error", reject);
-			fileStream.on("end", () => resolve(Buffer.concat(chunks)));
-		});
-
-		return fileBuffer;
+	const buffer = await new Promise<Buffer>((resolve, reject) => {
+		const chunks: Uint8Array[] = [];
+		fileStream.on("data", (chunk) => chunks.push(chunk));
+		fileStream.on("error", reject);
+		fileStream.on("end", () => resolve(Buffer.concat(chunks)));
 	});
 
-	return { type: "base64", value: buffer.toString("base64"), mime };
+	return { type: "base64", name, value: buffer.toString("base64"), mime };
 }

src/lib/server/files/uploadFile.ts

@@ -20,7 +20,9 @@ export async function uploadFile(file: File, conv: Conversation): Promise<Messag
 
 	// only return the filename when upload throws a finish event or a 20s time out occurs
 	return new Promise((resolve, reject) => {
-		upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type }));
+		upload.once("finish", () =>
+			resolve({ type: "hash", value: sha, mime: file.type, name: file.name })
+		);
 		upload.once("error", reject);
 		setTimeout(() => reject(new Error("Upload timed out")), 20_000);
 	});

src/lib/server/models.ts

@@ -12,6 +12,7 @@ import type { PreTrainedTokenizer } from "@xenova/transformers";
 import JSON5 from "json5";
 import { getTokenizer } from "$lib/utils/getTokenizer";
 import { logger } from "$lib/server/logger";
+import { ToolResultStatus } from "$lib/types/Tool";
 
 type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
 
@@ -61,6 +62,7 @@ const modelConfig = z.object({
 		.passthrough()
 		.optional(),
 	multimodal: z.boolean().default(false),
+	tools: z.boolean().default(false),
 	unlisted: z.boolean().default(false),
 	embeddingModel: validateEmbeddingModelByName(embeddingModels).optional(),
 });
@@ -94,7 +96,7 @@ async function getChatPromptRender(
 		process.exit();
 	}
 
-	const renderTemplate = ({ messages, preprompt }: ChatTemplateInput) => {
+	const renderTemplate = ({ messages, preprompt, tools, toolResults }: ChatTemplateInput) => {
 		let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({
 			content: message.content,
 			role: message.from,
@@ -110,9 +112,64 @@ async function getChatPromptRender(
 			];
 		}
 
+		if (toolResults?.length) {
+			// todo: should update the command r+ tokenizer to support system messages at any location
+			// or use the `rag` mode without the citations
+			formattedMessages = [
+				{
+					role: "system",
+					content:
+						"\n\n<results>\n" +
+						toolResults
+							.flatMap((result, idx) => {
+								if (result.status === ToolResultStatus.Error) {
+									return (
+										`Document: ${idx}\n` + `Tool "${result.call.name}" error\n` + result.message
+									);
+								}
+								return (
+									`Document: ${idx}\n` +
+									result.outputs
+										.flatMap((output) =>
+											Object.entries(output).map(([title, text]) => `${title}\n${text}`)
+										)
+										.join("\n")
+								);
+							})
+							.join("\n\n") +
+						"\n</results>",
+				},
+				...formattedMessages,
+			];
+			tools = [];
+		}
+
+		const chatTemplate = tools?.length ? "tool_use" : undefined;
+
+		const documents = (toolResults ?? []).flatMap((result) => {
+			if (result.status === ToolResultStatus.Error) {
+				return [{ title: `Tool "${result.call.name}" error`, text: "\n" + result.message }];
+			}
+			return result.outputs.flatMap((output) =>
+				Object.entries(output).map(([title, text]) => ({
+					title: `Tool "${result.call.name}" ${title}`,
+					text: "\n" + text,
+				}))
+			);
+		});
+
 		const output = tokenizer.apply_chat_template(formattedMessages, {
 			tokenize: false,
 			add_generation_prompt: true,
+			chat_template: chatTemplate,
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			tools:
+				tools?.map(({ parameterDefinitions, ...tool }) => ({
+					parameter_definitions: parameterDefinitions,
+					...tool,
+				})) ?? [],
+			documents,
 		});
 
 		if (typeof output !== "string") {
@@ -134,6 +191,10 @@ const processModel = async (m: z.infer<typeof modelConfig>) => ({
 	parameters: { ...m.parameters, stop_sequences: m.parameters?.stop },
 });
 
+export type ProcessedModel = Awaited<ReturnType<typeof processModel>> & {
+	getEndpoint: () => Promise<Endpoint>;
+};
+
 const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
 	...m,
 	getEndpoint: async (): Promise<Endpoint> => {
@@ -189,7 +250,9 @@ const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
 	},
 });
 
-export const models = await Promise.all(modelsRaw.map((e) => processModel(e).then(addEndpoint)));
+export const models: ProcessedModel[] = await Promise.all(
+	modelsRaw.map((e) => processModel(e).then(addEndpoint))
+);
 
 export const defaultModel = models[0];
 
@@ -224,5 +287,5 @@ export const smallModel = env.TASK_MODEL
 
 export type BackendModel = Optional<
 	typeof defaultModel,
-	"preprompt" | "parameters" | "multimodal" | "unlisted"
+	"preprompt" | "parameters" | "multimodal" | "unlisted" | "tools"
 >;

src/lib/server/textGeneration/assistant.ts

@@ -0,0 +1,53 @@
+import { isURLLocal } from "../isURLLocal";
+import { env } from "$env/dynamic/private";
+import { collections } from "$lib/server/database";
+import type { Assistant } from "$lib/types/Assistant";
+import type { ObjectId } from "mongodb";
+
+export async function processPreprompt(preprompt: string) {
+	const urlRegex = /{{\s?url=(.*?)\s?}}/g;
+
+	for (const match of preprompt.matchAll(urlRegex)) {
+		try {
+			const url = new URL(match[1]);
+			if ((await isURLLocal(url)) && env.ENABLE_LOCAL_FETCH !== "true") {
+				throw new Error("URL couldn't be fetched, it resolved to a local address.");
+			}
+
+			const res = await fetch(url.href);
+
+			if (!res.ok) {
+				throw new Error("URL couldn't be fetched, error " + res.status);
+			}
+			const text = await res.text();
+			preprompt = preprompt.replaceAll(match[0], text);
+		} catch (e) {
+			preprompt = preprompt.replaceAll(match[0], (e as Error).message);
+		}
+	}
+
+	return preprompt;
+}
+
+export async function getAssistantById(id?: ObjectId) {
+	return collections.assistants
+		.findOne<Pick<Assistant, "rag" | "dynamicPrompt" | "generateSettings">>(
+			{ _id: id },
+			{ projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1 } }
+		)
+		.then((a) => a ?? undefined);
+}
+
+export function assistantHasWebSearch(assistant?: Pick<Assistant, "rag"> | null) {
+	return (
+		env.ENABLE_ASSISTANTS_RAG === "true" &&
+		!!assistant?.rag &&
+		(assistant.rag.allowedLinks.length > 0 ||
+			assistant.rag.allowedDomains.length > 0 ||
+			assistant.rag.allowAllDomains)
+	);
+}
+
+export function assistantHasDynamicPrompt(assistant?: Pick<Assistant, "dynamicPrompt">) {
+	return env.ENABLE_ASSISTANTS_RAG === "true" && Boolean(assistant?.dynamicPrompt);
+}

src/lib/server/textGeneration/generate.ts

@@ -0,0 +1,51 @@
+import type { ToolResult } from "$lib/types/Tool";
+import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate";
+import { AbortedGenerations } from "../abortedGenerations";
+import type { TextGenerationContext } from "./types";
+import type { EndpointMessage } from "../endpoints/endpoints";
+
+type GenerateContext = Omit<TextGenerationContext, "messages"> & { messages: EndpointMessage[] };
+
+export async function* generate(
+	{ model, endpoint, conv, messages, assistant, isContinue, promptedAt }: GenerateContext,
+	toolResults: ToolResult[],
+	preprompt?: string
+): AsyncIterable<MessageUpdate> {
+	for await (const output of await endpoint({
+		messages,
+		preprompt,
+		continueMessage: isContinue,
+		generateSettings: assistant?.generateSettings,
+		toolResults,
+	})) {
+		// text generation completed
+		if (output.generated_text) {
+			let interrupted =
+				!output.token.special && !model.parameters.stop?.includes(output.token.text);
+
+			let text = output.generated_text.trimEnd();
+			for (const stopToken of model.parameters.stop ?? []) {
+				if (!text.endsWith(stopToken)) continue;
+
+				interrupted = false;
+				text = text.slice(0, text.length - stopToken.length);
+			}
+
+			yield { type: MessageUpdateType.FinalAnswer, text, interrupted };
+			continue;
+		}
+
+		// ignore special tokens
+		if (output.token.special) continue;
+
+		// pass down normal token
+		yield { type: MessageUpdateType.Stream, token: output.token.text };
+
+		// abort check
+		const date = AbortedGenerations.getInstance().getList().get(conv._id.toString());
+		if (date && date > promptedAt) break;
+
+		// no output check
+		if (!output) break;
+	}
+}

src/lib/server/textGeneration/index.ts

@@ -0,0 +1,64 @@
+import { runWebSearch } from "$lib/server/websearch/runWebSearch";
+import { preprocessMessages } from "../endpoints/preprocessMessages";
+
+import { generateTitleForConversation } from "./title";
+import {
+	assistantHasDynamicPrompt,
+	assistantHasWebSearch,
+	getAssistantById,
+	processPreprompt,
+} from "./assistant";
+import { pickTools, runTools } from "./tools";
+import type { WebSearch } from "$lib/types/WebSearch";
+import {
+	type MessageUpdate,
+	MessageUpdateType,
+	MessageUpdateStatus,
+} from "$lib/types/MessageUpdate";
+import { generate } from "./generate";
+import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
+import type { TextGenerationContext } from "./types";
+
+export async function* textGeneration(ctx: TextGenerationContext) {
+	yield* mergeAsyncGenerators([
+		textGenerationWithoutTitle(ctx),
+		generateTitleForConversation(ctx.conv),
+	]);
+}
+
+async function* textGenerationWithoutTitle(
+	ctx: TextGenerationContext
+): AsyncGenerator<MessageUpdate, undefined, undefined> {
+	yield {
+		type: MessageUpdateType.Status,
+		status: MessageUpdateStatus.Started,
+	};
+
+	ctx.assistant ??= await getAssistantById(ctx.conv.assistantId);
+	const { model, conv, messages, assistant, isContinue, webSearch, toolsPreference } = ctx;
+	const convId = conv._id;
+
+	// perform websearch if requested
+	// it can be because the user toggled the webSearch or because the assistant has webSearch enabled
+	// if tools are enabled, we don't perform it here since we will add the websearch as a tool
+	let webSearchResult: WebSearch | undefined;
+	if (
+		!isContinue &&
+		!model.tools &&
+		((webSearch && !conv.assistantId) || assistantHasWebSearch(assistant))
+	) {
+		webSearchResult = yield* runWebSearch(conv, messages, assistant?.rag);
+	}
+
+	let preprompt = conv.preprompt;
+	if (assistantHasDynamicPrompt(assistant) && preprompt) {
+		preprompt = await processPreprompt(preprompt);
+		if (messages[0].from === "system") messages[0].content = preprompt;
+	}
+
+	const tools = pickTools(toolsPreference, Boolean(assistant));
+	const toolResults = yield* runTools(ctx, tools, preprompt);
+
+	const processedMessages = await preprocessMessages(messages, webSearchResult, convId);
+	yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt);
+}

src/lib/server/{summarize.ts → textGeneration/title.ts}

@@ -1,14 +1,41 @@
 import { env } from "$env/dynamic/private";
 import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
-import type { EndpointMessage } from "./endpoints/endpoints";
+import type { EndpointMessage } from "../endpoints/endpoints";
 import { logger } from "$lib/server/logger";
+import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate";
+import type { Conversation } from "$lib/types/Conversation";
 
-export async function summarize(prompt: string) {
+export async function* generateTitleForConversation(
+	conv: Conversation
+): AsyncGenerator<MessageUpdate, undefined, undefined> {
+	try {
+		const userMessage = conv.messages.find((m) => m.from === "user");
+		// HACK: detect if the conversation is new
+		if (conv.title !== "New Chat" || !userMessage) return;
+
+		const prompt = userMessage.content;
+		const title = (await generateTitle(prompt)) ?? "New Chat";
+
+		yield {
+			type: MessageUpdateType.Title,
+			title,
+		};
+	} catch (cause) {
+		console.error(Error("Failed whilte generating title for conversation", { cause }));
+	}
+}
+
+export async function generateTitle(prompt: string) {
 	if (!env.LLM_SUMMERIZATION) {
 		return prompt.split(/\s+/g).slice(0, 5).join(" ");
 	}
 
 	const messages: Array<EndpointMessage> = [
+		{
+			from: "system",
+			content:
+				"You are a summarization AI. You'll never answer a user's question directly, but instead summarize the user's request into a single short sentence of four words or less. Always start your answer with an emoji relevant to the summary",
+		},
 		{ from: "user", content: "Who is the president of Gabon?" },
 		{ from: "assistant", content: "🇬🇦 President of Gabon" },
 		{ from: "user", content: "Who is Julien Chaumond?" },
@@ -23,6 +50,8 @@ export async function summarize(prompt: string) {
 		{ from: "assistant", content: "🎥 Favorite movie" },
 		{ from: "user", content: "Explain the concept of artificial intelligence in one sentence" },
 		{ from: "assistant", content: "🤖 AI definition" },
+		{ from: "user", content: "Draw a cute cat" },
+		{ from: "assistant", content: "🐱 Cute cat drawing" },
 		{ from: "user", content: prompt },
 	];
 

src/lib/server/textGeneration/tools.ts

@@ -0,0 +1,198 @@
+import { ToolResultStatus, type ToolCall, type ToolResult } from "$lib/types/Tool";
+import { v4 as uuidV4 } from "uuid";
+import JSON5 from "json5";
+import type { BackendTool, BackendToolContext } from "../tools";
+import {
+	MessageToolUpdateType,
+	MessageUpdateStatus,
+	MessageUpdateType,
+	type MessageUpdate,
+} from "$lib/types/MessageUpdate";
+import type { TextGenerationContext } from "./types";
+
+import { allTools } from "../tools";
+import directlyAnswer from "../tools/directlyAnswer";
+import websearch from "../tools/web/search";
+import { z } from "zod";
+import { logger } from "../logger";
+import { toolHasName } from "../tools/utils";
+import type { MessageFile } from "$lib/types/Message";
+import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
+
+function makeFilesPrompt(files: MessageFile[], fileMessageIndex: number): string {
+	if (files.length === 0) {
+		return "The user has not uploaded any files. Do not attempt to use any tools that require files";
+	}
+
+	const stringifiedFiles = files
+		.map(
+			(file, fileIndex) =>
+				`  - fileMessageIndex ${fileMessageIndex} | fileIndex ${fileIndex} | ${file.name} (${file.mime})`
+		)
+		.join("\n");
+	return `Attached ${files.length} file${files.length === 1 ? "" : "s"}:\n${stringifiedFiles}`;
+}
+
+export function pickTools(
+	toolsPreference: Record<string, boolean>,
+	isAssistant: boolean
+): BackendTool[] {
+	// if it's an assistant, only support websearch for now
+	if (isAssistant) return [directlyAnswer, websearch];
+
+	// filter based on tool preferences, add the tools that are on by default
+	return allTools.filter((el) => {
+		if (el.isLocked && el.isOnByDefault) return true;
+		return toolsPreference?.[el.name] ?? el.isOnByDefault;
+	});
+}
+
+async function* runTool(
+	{ conv, messages, preprompt, assistant }: BackendToolContext,
+	tools: BackendTool[],
+	call: ToolCall
+): AsyncGenerator<MessageUpdate, ToolResult | undefined, undefined> {
+	const uuid = uuidV4();
+
+	const tool = tools.find((el) => toolHasName(call.name, el));
+	if (!tool) {
+		return { call, status: ToolResultStatus.Error, message: `Could not find tool "${call.name}"` };
+	}
+
+	// Special case for directly_answer tool where we ignore
+	if (toolHasName(directlyAnswer.name, tool)) return;
+
+	yield {
+		type: MessageUpdateType.Tool,
+		subtype: MessageToolUpdateType.Call,
+		uuid,
+		call,
+	};
+	try {
+		const toolResult = yield* tool.call(call.parameters, {
+			conv,
+			messages,
+			preprompt,
+			assistant,
+		});
+		yield {
+			type: MessageUpdateType.Tool,
+			subtype: MessageToolUpdateType.Result,
+			uuid,
+			result: { ...toolResult, call } as ToolResult,
+		};
+		return { ...toolResult, call } as ToolResult;
+	} catch (cause) {
+		console.error(Error(`Failed while running tool ${call.name}`), { cause });
+		return {
+			call,
+			status: ToolResultStatus.Error,
+			message: cause instanceof Error ? cause.message : String(cause),
+		};
+	}
+}
+
+export async function* runTools(
+	{ endpoint, conv, messages, assistant }: TextGenerationContext,
+	tools: BackendTool[],
+	preprompt?: string
+): AsyncGenerator<MessageUpdate, ToolResult[], undefined> {
+	const calls: ToolCall[] = [];
+
+	const messagesWithFilesPrompt = messages.map((message, idx) => {
+		if (!message.files?.length) return message;
+		return {
+			...message,
+			content: `${message.content}\n${makeFilesPrompt(message.files, idx)}`,
+		};
+	});
+
+	// do the function calling bits here
+	for await (const output of await endpoint({
+		messages: messagesWithFilesPrompt,
+		preprompt,
+		generateSettings: assistant?.generateSettings,
+		tools,
+	})) {
+		// model natively supports tool calls
+		if (output.token.toolCalls) {
+			calls.push(...output.token.toolCalls);
+			continue;
+		}
+
+		// look for a code blocks of ```json and parse them
+		// if they're valid json, add them to the calls array
+		if (output.generated_text) {
+			const codeBlocks = Array.from(output.generated_text.matchAll(/```json\n(.*?)```/gs));
+			if (codeBlocks.length === 0) continue;
+
+			// grab only the capture group from the regex match
+			for (const [, block] of codeBlocks) {
+				try {
+					calls.push(
+						...JSON5.parse(block).filter(isExternalToolCall).map(externalToToolCall).filter(Boolean)
+					);
+				} catch (cause) {
+					// error parsing the calls
+					yield {
+						type: MessageUpdateType.Status,
+						status: MessageUpdateStatus.Error,
+						message: cause instanceof Error ? cause.message : String(cause),
+					};
+				}
+			}
+		}
+	}
+
+	const toolContext: BackendToolContext = { conv, messages, preprompt, assistant };
+	const toolResults: (ToolResult | undefined)[] = yield* mergeAsyncGenerators(
+		calls.map((call) => runTool(toolContext, tools, call))
+	);
+	return toolResults.filter((result): result is ToolResult => result !== undefined);
+}
+
+const externalToolCall = z.object({
+	tool_name: z.string(),
+	parameters: z.record(z.any()),
+});
+
+type ExternalToolCall = z.infer<typeof externalToolCall>;
+
+function isExternalToolCall(call: unknown): call is ExternalToolCall {
+	return externalToolCall.safeParse(call).success;
+}
+
+function externalToToolCall(call: ExternalToolCall): ToolCall | undefined {
+	// Convert - to _ since some models insist on using _ instead of -
+	const tool = allTools.find((tool) => toolHasName(call.tool_name, tool));
+	if (!tool) {
+		logger.debug(`Model requested tool that does not exist: "${call.tool_name}". Skipping tool...`);
+		return;
+	}
+
+	const parametersWithDefaults: Record<string, string> = {};
+
+	for (const [key, definition] of Object.entries(tool.parameterDefinitions)) {
+		const value = call.parameters[key];
+
+		// Required so ensure it's there, otherwise return undefined
+		if (definition.required) {
+			if (value === undefined) {
+				logger.debug(
+					`Model requested tool "${call.tool_name}" but was missing required parameter "${key}". Skipping tool...`
+				);
+				return;
+			}
+			parametersWithDefaults[key] = value;
+			continue;
+		}
+
+		// Optional so use default if not there
+		parametersWithDefaults[key] = value ?? definition.default;
+	}
+
+	return {
+		name: call.tool_name,
+		parameters: parametersWithDefaults,
+	};
+}

src/lib/server/textGeneration/types.ts

@@ -0,0 +1,17 @@
+import type { ProcessedModel } from "../models";
+import type { Endpoint } from "../endpoints/endpoints";
+import type { Conversation } from "$lib/types/Conversation";
+import type { Message } from "$lib/types/Message";
+import type { Assistant } from "$lib/types/Assistant";
+
+export interface TextGenerationContext {
+	model: ProcessedModel;
+	endpoint: Endpoint;
+	conv: Conversation;
+	messages: Message[];
+	assistant?: Pick<Assistant, "rag" | "dynamicPrompt" | "generateSettings">;
+	isContinue: boolean;
+	webSearch: boolean;
+	toolsPreference: Record<string, boolean>;
+	promptedAt: Date;
+}

src/lib/server/tools/calculator.ts

@@ -0,0 +1,37 @@
+import { ToolResultStatus } from "$lib/types/Tool";
+import type { BackendTool } from ".";
+import vm from "node:vm";
+
+const calculator: BackendTool = {
+	name: "query_calculator",
+	displayName: "Calculator",
+	description:
+		"A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, ** (power) and /, as well as parenthesis ().",
+	isOnByDefault: true,
+	parameterDefinitions: {
+		equation: {
+			description:
+				"The formula to evaluate. EXACTLY as you would plug into a calculator. No words, no letters, only numbers and operators. Letters will make the tool crash.",
+			type: "formula",
+			required: true,
+		},
+	},
+	async *call(params) {
+		try {
+			const blocks = String(params.equation).split("\n");
+			const query = blocks[blocks.length - 1].replace(/[^-()\d/*+.]/g, "");
+
+			return {
+				status: ToolResultStatus.Success,
+				outputs: [{ calculator: `${query} = ${vm.runInNewContext(query)}` }],
+			};
+		} catch (e) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "Invalid expression",
+			};
+		}
+	},
+};
+
+export default calculator;

src/lib/server/tools/directlyAnswer.ts

@@ -0,0 +1,20 @@
+import { ToolResultStatus } from "$lib/types/Tool";
+import type { BackendTool } from ".";
+
+const directlyAnswer: BackendTool = {
+	name: "directly_answer",
+	isOnByDefault: true,
+	isHidden: true,
+	isLocked: true,
+	description: "Use this tool to let the user know you wish to answer directly",
+	parameterDefinitions: {},
+	async *call() {
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [],
+			display: false,
+		};
+	},
+};
+
+export default directlyAnswer;

src/lib/server/tools/documentParser.ts

@@ -0,0 +1,69 @@
+import type { BackendTool } from ".";
+import { ToolResultStatus } from "$lib/types/Tool";
+import { callSpace } from "./utils";
+import { downloadFile } from "$lib/server/files/downloadFile";
+
+type PdfParserInput = [Blob /* pdf */, string /* filename */];
+type PdfParserOutput = [string /* markdown */, Record<string, unknown> /* metadata */];
+
+const documentParser: BackendTool = {
+	name: "document_parser",
+	displayName: "Document Parser",
+	description: "Use this tool to parse any document and get its content in markdown format.",
+	isOnByDefault: true,
+	parameterDefinitions: {
+		fileMessageIndex: {
+			description: "Index of the message containing the document file to parse",
+			type: "number",
+			required: true,
+		},
+		fileIndex: {
+			description: "Index of the document file to parse",
+			type: "number",
+			required: true,
+		},
+	},
+	async *call({ fileMessageIndex, fileIndex }, { conv, messages }) {
+		fileMessageIndex = Number(fileMessageIndex);
+		fileIndex = Number(fileIndex);
+
+		const message = messages[fileMessageIndex];
+		const files = message?.files ?? [];
+		if (!files || files.length === 0) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "User did not provide a pdf to parse",
+			};
+		}
+		if (fileIndex >= files.length) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "Model provided an invalid file index",
+			};
+		}
+
+		const file = files[fileIndex];
+		const fileBlob = await downloadFile(files[fileIndex].value, conv._id)
+			.then((file) => fetch(`data:${file.mime};base64,${file.value}`))
+			.then((res) => res.blob());
+
+		const outputs = await callSpace<PdfParserInput, PdfParserOutput>(
+			"huggingchat/document-parser",
+			"predict",
+			[fileBlob, file.name]
+		);
+
+		let documentMarkdown = outputs[0];
+		// TODO: quick fix for avoiding context limit. eventually should use the tokenizer
+		if (documentMarkdown.length > 30_000) {
+			documentMarkdown = documentMarkdown.slice(0, 30_000) + "\n\n... (truncated)";
+		}
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [{ [file.name]: documentMarkdown }],
+			display: false,
+		};
+	},
+};
+
+export default documentParser;

src/lib/server/tools/images/editing.ts

@@ -0,0 +1,107 @@
+import type { BackendTool } from "..";
+import { uploadFile } from "../../files/uploadFile";
+import { ToolResultStatus } from "$lib/types/Tool";
+import { MessageUpdateType } from "$lib/types/MessageUpdate";
+import { callSpace, type GradioImage } from "../utils";
+import { downloadFile } from "$lib/server/files/downloadFile";
+
+type ImageEditingInput = [
+	Blob /* image */,
+	string /* prompt */,
+	string /* negative prompt */,
+	number /* guidance scale */,
+	number /* steps */
+];
+type ImageEditingOutput = [GradioImage];
+
+const imageEditing: BackendTool = {
+	name: "image_editing",
+	displayName: "Image Editing",
+	description: "Use this tool to edit an image from a prompt.",
+	isOnByDefault: true,
+	parameterDefinitions: {
+		prompt: {
+			description:
+				"A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.",
+			type: "string",
+			required: true,
+		},
+		fileMessageIndex: {
+			description: "Index of the message containing the file to edit",
+			type: "number",
+			required: true,
+		},
+		fileIndex: {
+			description: "Index of the file to edit",
+			type: "number",
+			required: true,
+		},
+	},
+	async *call({ prompt, fileMessageIndex, fileIndex }, { conv, messages }) {
+		prompt = String(prompt);
+		fileMessageIndex = Number(fileMessageIndex);
+		fileIndex = Number(fileIndex);
+
+		const message = messages[fileMessageIndex];
+		const images = message?.files ?? [];
+		if (!images || images.length === 0) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "User did not provide an image to edit.",
+			};
+		}
+		if (fileIndex >= images.length) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "Model provided an invalid file index",
+			};
+		}
+		if (!images[fileIndex].mime.startsWith("image/")) {
+			return {
+				status: ToolResultStatus.Error,
+				message: "Model provided a file indx which is not an image",
+			};
+		}
+
+		// todo: should handle multiple images
+		const image = await downloadFile(images[fileIndex].value, conv._id)
+			.then((file) => fetch(`data:${file.mime};base64,${file.value}`))
+			.then((res) => res.blob());
+
+		const outputs = await callSpace<ImageEditingInput, ImageEditingOutput>(
+			"multimodalart/cosxl",
+			"run_edit",
+			[
+				image,
+				prompt,
+				"", // negative prompt
+				7, // guidance scale
+				20, // steps
+			]
+		);
+
+		const outputImage = await fetch(outputs[0].url)
+			.then((res) => res.blob())
+			.then((blob) => new File([blob], outputs[0].orig_name, { type: blob.type }))
+			.then((file) => uploadFile(file, conv));
+
+		yield {
+			type: MessageUpdateType.File,
+			name: outputImage.name,
+			sha: outputImage.value,
+			mime: outputImage.mime,
+		};
+
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [
+				{
+					imageEditing: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`,
+				},
+			],
+			display: false,
+		};
+	},
+};
+
+export default imageEditing;

src/lib/server/tools/images/generation.ts

@@ -0,0 +1,92 @@
+import type { BackendTool } from "..";
+import { uploadFile } from "../../files/uploadFile";
+import { ToolResultStatus } from "$lib/types/Tool";
+import { MessageUpdateType } from "$lib/types/MessageUpdate";
+import { callSpace, type GradioImage } from "../utils";
+
+type ImageGenerationInput = [
+	number /* number (numeric value between 1 and 8) in 'Number of Images' Slider component */,
+	number /* number in 'Image Height' Number component */,
+	number /* number in 'Image Width' Number component */,
+	string /* prompt */,
+	number /* seed random */
+];
+type ImageGenerationOutput = [{ image: GradioImage }[]];
+
+const imageGeneration: BackendTool = {
+	name: "image_generation",
+	displayName: "Image Generation",
+	description: "Use this tool to generate an image from a prompt.",
+	isOnByDefault: true,
+	parameterDefinitions: {
+		prompt: {
+			description:
+				"A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.",
+			type: "string",
+			required: true,
+		},
+		numberOfImages: {
+			description: "Number of images to generate, between 1 and 8.",
+			type: "number",
+			required: false,
+			default: 1,
+		},
+		width: {
+			description: "Width of the generated image.",
+			type: "number",
+			required: false,
+			default: 1024,
+		},
+		height: {
+			description: "Height of the generated image.",
+			type: "number",
+			required: false,
+			default: 1024,
+		},
+	},
+	async *call({ prompt, numberOfImages }, { conv }) {
+		const outputs = await callSpace<ImageGenerationInput, ImageGenerationOutput>(
+			"ByteDance/Hyper-SDXL-1Step-T2I",
+			"/process_image",
+			[
+				Number(numberOfImages), // number (numeric value between 1 and 8) in 'Number of Images' Slider component
+				512, // number in 'Image Height' Number component
+				512, // number in 'Image Width' Number component
+				String(prompt), // prompt
+				Math.floor(Math.random() * 1000), // seed random
+			]
+		);
+		const imageBlobs = await Promise.all(
+			outputs[0].map((output) =>
+				fetch(output.image.url)
+					.then((res) => res.blob())
+					.then(
+						(blob) =>
+							new File([blob], `${prompt}.${blob.type.split("/")[1] ?? "png"}`, { type: blob.type })
+					)
+					.then((file) => uploadFile(file, conv))
+			)
+		);
+
+		for (const image of imageBlobs) {
+			yield {
+				type: MessageUpdateType.File,
+				name: image.name,
+				sha: image.value,
+				mime: image.mime,
+			};
+		}
+
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [
+				{
+					imageGeneration: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`,
+				},
+			],
+			display: false,
+		};
+	},
+};
+
+export default imageGeneration;

src/lib/server/tools/index.ts

@@ -0,0 +1,37 @@
+import type { Assistant } from "$lib/types/Assistant";
+import type { Conversation } from "$lib/types/Conversation";
+import type { Message } from "$lib/types/Message";
+import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import type { Tool, ToolResult } from "$lib/types/Tool";
+
+import calculator from "./calculator";
+import directlyAnswer from "./directlyAnswer";
+import imageEditing from "./images/editing";
+import imageGeneration from "./images/generation";
+import documentParser from "./documentParser";
+import fetchUrl from "./web/url";
+import websearch from "./web/search";
+
+export interface BackendToolContext {
+	conv: Conversation;
+	messages: Message[];
+	preprompt?: string;
+	assistant?: Pick<Assistant, "rag" | "dynamicPrompt" | "generateSettings">;
+}
+
+export interface BackendTool extends Tool {
+	call(
+		params: Record<string, string | number | boolean>,
+		context: BackendToolContext
+	): AsyncGenerator<MessageUpdate, Omit<ToolResult, "call">, undefined>;
+}
+
+export const allTools: BackendTool[] = [
+	directlyAnswer,
+	websearch,
+	imageGeneration,
+	fetchUrl,
+	imageEditing,
+	documentParser,
+	calculator,
+];

src/lib/server/tools/utils.ts

@@ -0,0 +1,29 @@
+import { env } from "$env/dynamic/private";
+import { Client } from "@gradio/client";
+
+export type GradioImage = {
+	path: string;
+	url: string;
+	orig_name: string;
+	is_stream: boolean;
+	meta: Record<string, unknown>;
+};
+
+type GradioResponse = {
+	data: unknown[];
+};
+
+export async function callSpace<TInput extends unknown[], TOutput extends unknown[]>(
+	name: string,
+	func: string,
+	parameters: TInput
+): Promise<TOutput> {
+	const client = await Client.connect(name, {
+		hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`,
+	});
+	return await client
+		.predict(func, parameters)
+		.then((res) => (res as unknown as GradioResponse).data as TOutput);
+}
+
+export { toolHasName } from "$lib/utils/tools";

src/lib/server/tools/web/search.ts

@@ -0,0 +1,33 @@
+import { ToolResultStatus } from "$lib/types/Tool";
+import type { BackendTool } from "..";
+import { runWebSearch } from "../../websearch/runWebSearch";
+
+const websearch: BackendTool = {
+	name: "websearch",
+	displayName: "Web Search",
+	isOnByDefault: true,
+	description:
+		"Use this tool to search web pages for answers that will help answer the user's query. Only use this tool if you need specific resources from the internet.",
+	parameterDefinitions: {
+		query: {
+			required: true,
+			type: "string",
+			description:
+				"A search query which will be used to fetch the most relevant snippets regarding the user's query",
+		},
+	},
+	async *call({ query }, { conv, assistant, messages }) {
+		const webSearchToolResults = yield* runWebSearch(conv, messages, assistant?.rag, String(query));
+		const chunks = webSearchToolResults?.contextSources
+			.map(({ context }) => context)
+			.join("\n------------\n");
+
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [{ websearch: chunks }],
+			display: false,
+		};
+	},
+};
+
+export default websearch;

src/lib/server/tools/web/url.ts

@@ -0,0 +1,32 @@
+import { stringifyMarkdownElementTree } from "$lib/server/websearch/markdown/utils/stringify";
+import { scrapeUrl } from "$lib/server/websearch/scrape/scrape";
+import { ToolResultStatus } from "$lib/types/Tool";
+import type { BackendTool } from "..";
+
+const fetchUrl: BackendTool = {
+	name: "fetch_url",
+	displayName: "URL Fetcher",
+	description: "A tool that can be used to fetch an URL and return the content directly.",
+	isOnByDefault: true,
+	parameterDefinitions: {
+		url: {
+			description: "The url that should be fetched.",
+			type: "str",
+			required: true,
+		},
+	},
+	async *call(params) {
+		const blocks = String(params.url).split("\n");
+		const url = blocks[blocks.length - 1];
+
+		const { title, markdownTree } = await scrapeUrl(url, Infinity);
+
+		return {
+			status: ToolResultStatus.Success,
+			outputs: [{ title, text: stringifyMarkdownElementTree(markdownTree) }],
+			display: false,
+		};
+	},
+};
+
+export default fetchUrl;

src/lib/server/websearch/markdown/utils/stringify.ts

@@ -26,6 +26,13 @@ export function stringifyMarkdownElement(elem: MarkdownElement): string {
 	return `${content}\n\n`;
 }
 
+/** Converts a tree of markdown elements to a string with formatting */
+export function stringifyMarkdownElementTree(elem: MarkdownElement): string {
+	const stringified = stringifyMarkdownElement(elem);
+	if (!("children" in elem)) return stringified;
+	return stringified + elem.children.map(stringifyMarkdownElementTree).join("");
+}
+
 // ----- HTML Elements -----
 
 /** Ignores all non-inline tag types and grabs their text. Converts inline tags to markdown */

src/lib/server/websearch/runWebSearch.ts

@@ -1,35 +1,35 @@
 import { defaultEmbeddingModel, embeddingModels } from "$lib/server/embeddingModels";
 
 import type { Conversation } from "$lib/types/Conversation";
-import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import type { Message } from "$lib/types/Message";
 import type { WebSearch, WebSearchScrapedSource } from "$lib/types/WebSearch";
 import type { Assistant } from "$lib/types/Assistant";
+import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate";
 
 import { search } from "./search/search";
 import { scrape } from "./scrape/scrape";
 import { findContextSources } from "./embed/embed";
 import { removeParents } from "./markdown/tree";
+import {
+	makeErrorUpdate,
+	makeFinalAnswerUpdate,
+	makeGeneralUpdate,
+	makeSourcesUpdate,
+} from "./update";
+import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
 
 const MAX_N_PAGES_TO_SCRAPE = 8 as const;
 const MAX_N_PAGES_TO_EMBED = 5 as const;
 
-export type AppendUpdate = (message: string, args?: string[], type?: "error" | "update") => void;
-const makeAppendUpdate =
-	(updatePad: (upd: MessageUpdate) => void): AppendUpdate =>
-	(message, args, type) =>
-		updatePad({ type: "webSearch", messageType: type ?? "update", message, args });
-
-export async function runWebSearch(
+export async function* runWebSearch(
 	conv: Conversation,
 	messages: Message[],
-	updatePad: (upd: MessageUpdate) => void,
-	ragSettings?: Assistant["rag"]
-): Promise<WebSearch> {
+	ragSettings?: Assistant["rag"],
+	query?: string
+): AsyncGenerator<MessageWebSearchUpdate, WebSearch, undefined> {
 	const prompt = messages[messages.length - 1].content;
 	const createdAt = new Date();
 	const updatedAt = new Date();
-	const appendUpdate = makeAppendUpdate(updatePad);
 
 	try {
 		const embeddingModel =
@@ -39,29 +39,26 @@ export async function runWebSearch(
 		}
 
 		// Search the web
-		const { searchQuery, pages } = await search(messages, ragSettings, appendUpdate);
+		const { searchQuery, pages } = yield* search(messages, ragSettings, query);
 		if (pages.length === 0) throw Error("No results found for this search query");
 
 		// Scrape pages
-		appendUpdate("Browsing search results");
+		yield makeGeneralUpdate({ message: "Browsing search results" });
 
-		const scrapedPages = await Promise.all(
-			pages
-				.slice(0, MAX_N_PAGES_TO_SCRAPE)
-				.map(scrape(appendUpdate, embeddingModel.chunkCharLength))
-		).then((allScrapedPages) =>
-			allScrapedPages
-				.filter((p): p is WebSearchScrapedSource => Boolean(p))
-				.filter((p) => p.page.markdownTree.children.length > 0)
-				.slice(0, MAX_N_PAGES_TO_EMBED)
+		const allScrapedPages = yield* mergeAsyncGenerators(
+			pages.slice(0, MAX_N_PAGES_TO_SCRAPE).map(scrape(embeddingModel.chunkCharLength))
 		);
+		const scrapedPages = allScrapedPages
+			.filter((p): p is WebSearchScrapedSource => Boolean(p))
+			.filter((p) => p.page.markdownTree.children.length > 0)
+			.slice(0, MAX_N_PAGES_TO_EMBED);
 
 		if (!scrapedPages.length) {
 			throw Error(`No text found in the first ${MAX_N_PAGES_TO_SCRAPE} results`);
 		}
 
 		// Chunk the text of each of the elements and find the most similar chunks to the prompt
-		appendUpdate("Extracting relevant information");
+		yield makeGeneralUpdate({ message: "Extracting relevant information" });
 		const contextSources = await findContextSources(scrapedPages, prompt, embeddingModel).then(
 			(ctxSources) =>
 				ctxSources.map((source) => ({
@@ -69,14 +66,9 @@ export async function runWebSearch(
 					page: { ...source.page, markdownTree: removeParents(source.page.markdownTree) },
 				}))
 		);
-		updatePad({
-			type: "webSearch",
-			messageType: "sources",
-			message: "sources",
-			sources: contextSources,
-		});
+		yield makeSourcesUpdate(contextSources);
 
-		return {
+		const webSearch: WebSearch = {
 			prompt,
 			searchQuery,
 			results: scrapedPages.map(({ page, ...source }) => ({
@@ -87,11 +79,14 @@ export async function runWebSearch(
 			createdAt,
 			updatedAt,
 		};
+		yield makeFinalAnswerUpdate(webSearch);
+		return webSearch;
 	} catch (searchError) {
 		const message = searchError instanceof Error ? searchError.message : String(searchError);
 		console.error(message);
-		appendUpdate("An error occurred", [JSON.stringify(message)], "error");
-		return {
+		yield makeErrorUpdate({ message: "An error occurred", args: [message] });
+
+		const webSearch: WebSearch = {
 			prompt,
 			searchQuery: "",
 			results: [],
@@ -99,5 +94,7 @@ export async function runWebSearch(
 			createdAt,
 			updatedAt,
 		};
+		yield makeFinalAnswerUpdate(webSearch);
+		return webSearch;
 	}
 }

src/lib/server/websearch/scrape/playwright.ts

@@ -4,6 +4,7 @@ import {
 	devices,
 	type Page,
 	type BrowserContextOptions,
+	type Response,
 } from "playwright";
 import { PlaywrightBlocker } from "@cliqz/adblocker-playwright";
 import { env } from "$env/dynamic/private";
@@ -44,16 +45,16 @@ async function initPlaywrightService() {
 	return Object.freeze({ ctx, blocker });
 }
 
-export async function loadPage(url: string): Promise<Page> {
+export async function loadPage(url: string): Promise<{ res?: Response; page: Page }> {
 	if (!playwrightService) playwrightService = initPlaywrightService();
 	const { ctx, blocker } = await playwrightService;
 
 	const page = await ctx.newPage();
 	await blocker.enableBlockingInPage(page);
 
-	await page.goto(url, { waitUntil: "load", timeout: 2000 }).catch(() => {
+	const res = await page.goto(url, { waitUntil: "load", timeout: 3500 }).catch(() => {
 		console.warn(`Failed to load page within 2s: ${url}`);
 	});
 
-	return page;
+	return { res: res ?? undefined, page };
 }

src/lib/server/websearch/scrape/scrape.ts

@@ -1,26 +1,52 @@
-import type { AppendUpdate } from "../runWebSearch";
 import type { WebSearchScrapedSource, WebSearchSource } from "$lib/types/WebSearch";
+import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate";
 import { loadPage } from "./playwright";
 
 import { spatialParser } from "./parser";
 import { htmlToMarkdownTree } from "../markdown/tree";
 import { timeout } from "$lib/utils/timeout";
+import { makeErrorUpdate, makeGeneralUpdate } from "../update";
 
-export const scrape =
-	(appendUpdate: AppendUpdate, maxCharsPerElem: number) =>
-	async (source: WebSearchSource): Promise<WebSearchScrapedSource | undefined> => {
+export const scrape = (maxCharsPerElem: number) =>
+	async function* (
+		source: WebSearchSource
+	): AsyncGenerator<MessageWebSearchUpdate, WebSearchScrapedSource | undefined, undefined> {
 		try {
 			const page = await scrapeUrl(source.link, maxCharsPerElem);
-			appendUpdate("Browsing webpage", [source.link]);
+			yield makeGeneralUpdate({ message: "Browsing webpage", args: [source.link] });
 			return { ...source, page };
 		} catch (e) {
 			const message = e instanceof Error ? e.message : String(e);
-			appendUpdate("Failed to parse webpage", [message, source.link], "error");
+			yield makeErrorUpdate({ message: "Failed to parse webpage", args: [message, source.link] });
 		}
 	};
 
 export async function scrapeUrl(url: string, maxCharsPerElem: number) {
-	const page = await loadPage(url);
+	const { res, page } = await loadPage(url);
+
+	if (!res) throw Error("Failed to load page");
+
+	// Check if it's a non-html content type that we can handle directly
+	// TODO: direct mappings to markdown can be added for markdown, csv and others
+	const contentType = res.headers()["content-type"] ?? "";
+	if (
+		contentType.includes("text/plain") ||
+		contentType.includes("text/markdown") ||
+		contentType.includes("application/json") ||
+		contentType.includes("application/xml") ||
+		contentType.includes("text/csv")
+	) {
+		const title = await page.title();
+		const content = await page.content();
+		return {
+			title,
+			markdownTree: htmlToMarkdownTree(
+				title,
+				[{ tagName: "p", attributes: {}, content: [content] }],
+				maxCharsPerElem
+			),
+		};
+	}
 
 	return timeout(page.evaluate(spatialParser), 2000)
 		.then(({ elements, ...parsed }) => ({

src/lib/server/websearch/search/search.ts

@@ -1,7 +1,6 @@
 import type { WebSearchSource } from "$lib/types/WebSearch";
 import type { Message } from "$lib/types/Message";
 import type { Assistant } from "$lib/types/Assistant";
-import type { AppendUpdate } from "../runWebSearch";
 import { getWebSearchProvider, searchWeb } from "./endpoints";
 import { generateQuery } from "./generateQuery";
 import { isURLStringLocal } from "$lib/server/isURLLocal";
@@ -10,30 +9,36 @@ import { isURL } from "$lib/utils/isUrl";
 import z from "zod";
 import JSON5 from "json5";
 import { env } from "$env/dynamic/private";
+import { makeGeneralUpdate } from "../update";
+import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate";
 
 const listSchema = z.array(z.string()).default([]);
 const allowList = listSchema.parse(JSON5.parse(env.WEBSEARCH_ALLOWLIST));
 const blockList = listSchema.parse(JSON5.parse(env.WEBSEARCH_BLOCKLIST));
 
-export async function search(
+export async function* search(
 	messages: Message[],
-	ragSettings: Assistant["rag"] | undefined,
-	appendUpdate: AppendUpdate
-): Promise<{ searchQuery: string; pages: WebSearchSource[] }> {
+	ragSettings?: Assistant["rag"],
+	query?: string
+): AsyncGenerator<
+	MessageWebSearchUpdate,
+	{ searchQuery: string; pages: WebSearchSource[] },
+	undefined
+> {
 	if (ragSettings && ragSettings?.allowedLinks.length > 0) {
-		appendUpdate("Using links specified in Assistant");
+		yield makeGeneralUpdate({ message: "Using links specified in Assistant" });
 		return {
 			searchQuery: "",
 			pages: await directLinksToSource(ragSettings.allowedLinks).then(filterByBlockList),
 		};
 	}
 
-	const searchQuery = await generateQuery(messages);
-	appendUpdate(`Searching ${getWebSearchProvider()}`, [searchQuery]);
+	const searchQuery = query ?? (await generateQuery(messages));
+	yield makeGeneralUpdate({ message: `Searching ${getWebSearchProvider()}`, args: [searchQuery] });
 
 	// handle the global and (optional) rag lists
 	if (ragSettings && ragSettings?.allowedDomains.length > 0) {
-		appendUpdate("Filtering on specified domains");
+		yield makeGeneralUpdate({ message: "Filtering on specified domains" });
 	}
 	const filters = buildQueryFromSiteFilters(
 		[...(ragSettings?.allowedDomains ?? []), ...allowList],

src/lib/server/websearch/update.ts

@@ -0,0 +1,46 @@
+import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch";
+import {
+	MessageUpdateType,
+	MessageWebSearchUpdateType,
+	type MessageWebSearchErrorUpdate,
+	type MessageWebSearchFinishedUpdate,
+	type MessageWebSearchGeneralUpdate,
+	type MessageWebSearchSourcesUpdate,
+} from "$lib/types/MessageUpdate";
+
+export function makeGeneralUpdate(
+	update: Pick<MessageWebSearchGeneralUpdate, "message" | "args">
+): MessageWebSearchGeneralUpdate {
+	return {
+		type: MessageUpdateType.WebSearch,
+		subtype: MessageWebSearchUpdateType.Update,
+		...update,
+	};
+}
+
+export function makeErrorUpdate(
+	update: Pick<MessageWebSearchErrorUpdate, "message" | "args">
+): MessageWebSearchErrorUpdate {
+	return {
+		type: MessageUpdateType.WebSearch,
+		subtype: MessageWebSearchUpdateType.Error,
+		...update,
+	};
+}
+
+export function makeSourcesUpdate(sources: WebSearchSource[]): MessageWebSearchSourcesUpdate {
+	return {
+		type: MessageUpdateType.WebSearch,
+		subtype: MessageWebSearchUpdateType.Sources,
+		message: "sources",
+		sources,
+	};
+}
+
+export function makeFinalAnswerUpdate(webSearch: WebSearch): MessageWebSearchFinishedUpdate {
+	return {
+		type: MessageUpdateType.WebSearch,
+		subtype: MessageWebSearchUpdateType.Finished,
+		webSearch,
+	};
+}

src/lib/stores/settings.ts

@@ -15,6 +15,7 @@ type SettingsStore = {
 	customPrompts: Record<string, string>;
 	recentlySaved: boolean;
 	assistants: Array<ObjectId | string>;
+	tools?: Record<string, boolean>;
 };
 
 type SettingsStoreWritable = Writable<SettingsStore> & {

src/lib/types/Message.ts

@@ -27,6 +27,7 @@ export type Message = Partial<Timestamps> & {
 
 export type MessageFile = {
 	type: "hash" | "base64";
+	name: string;
 	value: string;
 	mime: string;
 };

src/lib/types/MessageUpdate.ts

@@ -1,46 +1,111 @@
-import type { WebSearchSource } from "./WebSearch";
+import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch";
+import type { ToolCall, ToolResult } from "$lib/types/Tool";
 
-export type FinalAnswer = {
-	type: "finalAnswer";
-	text: string;
-};
+export type MessageUpdate =
+	| MessageStatusUpdate
+	| MessageTitleUpdate
+	| MessageToolUpdate
+	| MessageWebSearchUpdate
+	| MessageStreamUpdate
+	| MessageFileUpdate
+	| MessageFinalAnswerUpdate;
 
-export type TextStreamUpdate = {
-	type: "stream";
-	token: string;
-};
+export enum MessageUpdateType {
+	Status = "status",
+	Title = "title",
+	Tool = "tool",
+	WebSearch = "webSearch",
+	Stream = "stream",
+	File = "file",
+	FinalAnswer = "finalAnswer",
+}
 
-export type AgentUpdate = {
-	type: "agent";
-	agent: string;
-	content: string;
-	binary?: Blob;
-};
+// Status
+export enum MessageUpdateStatus {
+	Started = "started",
+	Error = "error",
+	Finished = "finished",
+}
+export interface MessageStatusUpdate {
+	type: MessageUpdateType.Status;
+	status: MessageUpdateStatus;
+	message?: string;
+}
 
-export type WebSearchUpdate = {
-	type: "webSearch";
-	messageType: "update" | "error" | "sources";
+// Web search
+export enum MessageWebSearchUpdateType {
+	Update = "update",
+	Error = "error",
+	Sources = "sources",
+	Finished = "finished",
+}
+interface BaseMessageWebSearchUpdate<TSubType extends MessageWebSearchUpdateType> {
+	type: MessageUpdateType.WebSearch;
+	subtype: TSubType;
+}
+export interface MessageWebSearchErrorUpdate
+	extends BaseMessageWebSearchUpdate<MessageWebSearchUpdateType.Error> {
 	message: string;
 	args?: string[];
-	sources?: WebSearchSource[];
-};
+}
+export interface MessageWebSearchGeneralUpdate
+	extends BaseMessageWebSearchUpdate<MessageWebSearchUpdateType.Update> {
+	message: string;
+	args?: string[];
+}
+export interface MessageWebSearchSourcesUpdate
+	extends BaseMessageWebSearchUpdate<MessageWebSearchUpdateType.Sources> {
+	message: string;
+	sources: WebSearchSource[];
+}
+export interface MessageWebSearchFinishedUpdate
+	extends BaseMessageWebSearchUpdate<MessageWebSearchUpdateType.Finished> {
+	webSearch: WebSearch;
+}
+export type MessageWebSearchUpdate =
+	| MessageWebSearchErrorUpdate
+	| MessageWebSearchGeneralUpdate
+	| MessageWebSearchSourcesUpdate
+	| MessageWebSearchFinishedUpdate;
 
-export type StatusUpdate = {
-	type: "status";
-	status: "started" | "pending" | "finished" | "error" | "title";
-	message?: string;
-};
+// Tool
+export enum MessageToolUpdateType {
+	/** A request to call a tool alongside it's parameters */
+	Call = "call",
+	/** The result of a tool call */
+	Result = "result",
+}
+interface MessageToolBaseUpdate<TSubType extends MessageToolUpdateType> {
+	type: MessageUpdateType.Tool;
+	subtype: TSubType;
+	uuid: string;
+}
+export interface MessageToolCallUpdate extends MessageToolBaseUpdate<MessageToolUpdateType.Call> {
+	call: ToolCall;
+}
+export interface MessageToolResultUpdate
+	extends MessageToolBaseUpdate<MessageToolUpdateType.Result> {
+	result: ToolResult;
+}
+export type MessageToolUpdate = MessageToolCallUpdate | MessageToolResultUpdate;
 
-export type ErrorUpdate = {
-	type: "error";
-	message: string;
+// Everything else
+export interface MessageTitleUpdate {
+	type: MessageUpdateType.Title;
+	title: string;
+}
+export interface MessageStreamUpdate {
+	type: MessageUpdateType.Stream;
+	token: string;
+}
+export interface MessageFileUpdate {
+	type: MessageUpdateType.File;
 	name: string;
-};
-
-export type MessageUpdate =
-	| FinalAnswer
-	| TextStreamUpdate
-	| AgentUpdate
-	| WebSearchUpdate
-	| StatusUpdate
-	| ErrorUpdate;
+	sha: string;
+	mime: string;
+}
+export interface MessageFinalAnswerUpdate {
+	type: MessageUpdateType.FinalAnswer;
+	text: string;
+	interrupted: boolean;
+}

src/lib/types/Model.ts

@@ -17,4 +17,5 @@ export type Model = Pick<
 	| "preprompt"
 	| "multimodal"
 	| "unlisted"
+	| "tools"
 >;

src/lib/types/Settings.ts

@@ -21,6 +21,7 @@ export interface Settings extends Timestamps {
 	customPrompts?: Record<string, string>;
 
 	assistants?: Assistant["_id"][];
+	tools?: Record<string, boolean>;
 }
 
 // TODO: move this to a constant file along with other constants
@@ -30,4 +31,5 @@ export const DEFAULT_SETTINGS = {
 	hideEmojiOnSidebar: false,
 	customPrompts: {},
 	assistants: [],
+	tools: {},
 };