Community tools (#1250)

* wip community tools

* push

* fix

* bugfix

* pass inputs to gradio correctly

* full inputs for imagegen

* lint

* update gradio

* fix call to gradio

* fix imagegen tool

* fix name

* fix image display

* Community tools (#1297)

* work on tools

* wip

* icon

* page

* use IDs for every tools

* improve tools page

* wip

* add optimized deps

* refacto 1tool=1function

* Add preview page

* fix active indicator

* fix text alignment

* fix populate script

* tooledit

* fixes

* better inputs

* fix console error & cancel button

* upload new tools

* edit

* improve lint

* working community tools

* fix active featured check

* Simplify outputs in user form

* fix non file outputs

* fix imagegen default tool

* add community tools popup

* redirect gradio API calls through chat-ui backend

* better usecount

* Better loading & output element

* better redirects & auto activate

* bugfixes

* fix migration changes

* Add feature flag for tools

* fix tools menu

* Keep track of output component index

* fix null check

* catch error for spaces config

* wip: MIME types

* wip: rework tools to use filenames

* fix broken tool calling

* fix input tools

* Working files

* fix(form): bind on name

* fix(prompt): double system prompt

* fix(prompt): file list

* fix(tools): make sure tools are off by default

* Add description to ToolEdit

* Display runs

* feat(tools): add back document parser

* feat(tools): bring back image editing tool

* fix MIME types

* Fix file passing prompts

* fix lint

* Set tools to default in migration

* Revert "Set tools to default in migration"

This reverts commit 87fdd094b73e2427af1a272e6697dafd4d466f1c.

* Use correct space for official image generation

* Remove debug logs

* fix(redirect): working redirect when no `APP_BASE`

* sveltekit 2 migration

* fix types with gradio update

* fix name based searching

* fix cut titles

* add min & max

* add error if no endpoint

* make form not submittable until it's filled

* fix return

* Better support for varied input types

* fix(update): hide null values for params that have not been called explicitly by the model

* add extra migration for transition

* fix document parser mime types

.env

@@ -163,6 +163,9 @@ ALLOW_INSECURE_COOKIES=false # recommended to keep this to false but set to true
 METRICS_ENABLED=false
 METRICS_PORT=5565
 LOG_LEVEL=info
+
+
+TOOLS=`[]` 
 BODY_SIZE_LIMIT=15728640
 
 HF_ORG_ADMIN=

chart/env/prod.yaml

@@ -328,6 +328,151 @@ envVars:
     }]
   WEBSEARCH_BLOCKLIST: '["youtube.com", "twitter.com"]'
   XFF_DEPTH: '2'
+  TOOLS: >
+    [
+      {
+        "_id": "000000000000000000000001",
+        "displayName": "Image Generation",
+        "description": "Use this tool to generate images based on a prompt.",
+        "color": "yellow",
+        "icon": "camera",
+        "baseUrl": "stabilityai/stable-diffusion-3-medium",
+        "name": "image_generation",
+        "endpoint": "/infer",
+        "inputs": [
+          {
+            "name": "prompt",
+            "description": "A prompt to generate an image from",
+            "paramType": "required",
+            "type": "str"
+          },
+          {
+            "name": "negative_prompt",
+            "description": "A prompt for things that should not be in the image.",
+            "paramType": "optional",
+            "default": "",
+            "type": "str"
+          },
+          {
+            "name": "seed",
+            "paramType": "fixed",
+            "value": "0",
+            "type": "float"
+          },
+          {
+            "name": "randomize_seed",
+            "paramType": "fixed",
+            "value": "true",
+            "type": "bool"
+          },
+          {
+            "name": "width",
+            "description": "numeric value between 256 and 1344",
+            "paramType": "optional",
+            "default": 1024,
+            "type": "float"
+          },
+          {
+            "name": "height",
+            "description": "numeric value between 256 and 1344",
+            "paramType": "optional",
+            "default": 1024,
+            "type": "float"
+          },
+          {
+            "name": "guidance_scale",
+            "description": "numeric value between 0.0 and 10.0",
+            "paramType": "optional",
+            "default": 5,
+            "type": "float"
+          },
+          {
+            "name": "num_inference_steps",
+            "description": "numeric value between 1 and 50",
+            "paramType": "optional",
+            "default": 28,
+            "type": "float"
+          }
+        ],
+        "outputComponent": "image",
+        "outputComponentIdx": 0,
+        "showOutput": true
+      },
+      {
+        "_id": "000000000000000000000002",
+        "displayName": "Document parser",
+        "description": "Use this tool to parse any document and get its content in markdown format.",
+        "color": "yellow",
+        "icon": "cloud",
+        "baseUrl": "huggingchat/document-parser",
+        "name": "document_parser",
+        "endpoint": "/predict",
+        "inputs": [
+          {
+            "name": "document",
+            "description": "Filename of the document to parse",
+            "paramType": "required",
+            "type": "file",
+            "mimeTypes": 'application/*'
+          },
+          {
+            "name": "filename",
+            "paramType": "fixed",
+            "value": "document.pdf",
+            "type": "str"
+          }
+        ],
+        "outputComponent": "textbox",
+        "outputComponentIdx": 0,
+        "showOutput": false
+      },
+      {
+        "_id": "000000000000000000000003",
+        "name": "edit_image",
+        "baseUrl": "multimodalart/cosxl",
+        "endpoint": "/run_edit",
+        "inputs": [
+          {
+            "name": "image",
+            "description": "The image path to be edited",
+            "paramType": "required",
+            "type": "file",
+            "mimeTypes": 'image/*'
+          },
+          {
+            "name": "prompt",
+            "description": "The prompt with which to edit the image",
+            "paramType": "required",
+            "type": "str"
+          },
+          {
+            "name": "negative_prompt",
+            "paramType": "fixed",
+            "value": "",
+            "type": "str"
+          },
+          {
+            "name": "guidance_scale",
+            "paramType": "fixed",
+            "value": 6.5,
+            "type": "float"
+          },
+          {
+            "name": "steps",
+            "paramType": "fixed",
+            "value": 30,
+            "type": "float"
+          }
+        ],
+        "outputComponent": "image",
+        "showOutput": true,
+        "displayName": "Image editor",
+        "color": "green",
+        "icon": "camera",
+        "description": "This tool lets you edit images",
+        "outputComponentIdx": 0
+      }
+    ]
   HF_ORG_ADMIN: 'huggingchat'
   HF_ORG_EARLY_ACCESS: 'huggingface'
 

package.json

@@ -27,6 +27,7 @@
 		"@types/express": "^4.17.21",
 		"@types/js-yaml": "^4.0.9",
 		"@types/jsdom": "^21.1.1",
+		"@types/jsonpath": "^0.2.4",
 		"@types/minimist": "^1.2.5",
 		"@types/node": "^22.1.0",
 		"@types/parquetjs": "^0.10.3",
@@ -59,7 +60,7 @@
 	"dependencies": {
 		"@aws-sdk/credential-providers": "^3.592.0",
 		"@cliqz/adblocker-playwright": "^1.27.2",
-		"@gradio/client": "^0.19.4",
+		"@gradio/client": "^1.1.1",
 		"@huggingface/hub": "^0.5.1",
 		"@huggingface/inference": "^2.7.0",
 		"@iconify-json/bi": "^1.1.21",
@@ -82,6 +83,7 @@
 		"jose": "^5.3.0",
 		"jsdom": "^22.0.0",
 		"json5": "^2.2.3",
+		"jsonpath": "^1.1.1",
 		"lint-staged": "^15.2.7",
 		"marked": "^12.0.1",
 		"marked-katex-extension": "^5.0.1",

scripts/populate.ts

@@ -14,6 +14,7 @@ import type { User } from "../src/lib/types/User";
 import type { Assistant } from "../src/lib/types/Assistant";
 import type { Conversation } from "../src/lib/types/Conversation";
 import type { Settings } from "../src/lib/types/Settings";
+import type { CommunityToolDB, ToolLogoColor, ToolLogoIcon } from "../src/lib/types/Tool";
 import { defaultEmbeddingModel } from "../src/lib/server/embeddingModels.ts";
 import { Message } from "../src/lib/types/Message.ts";
 
@@ -29,7 +30,7 @@ rl.on("close", function () {
 	process.exit(0);
 });
 
-const possibleFlags = ["reset", "all", "users", "settings", "assistants", "conversations"];
+const possibleFlags = ["reset", "all", "users", "settings", "assistants", "conversations", "tools"];
 const argv = minimist(process.argv.slice(2));
 const flags = argv["_"].filter((flag) => possibleFlags.includes(flag));
 
@@ -113,6 +114,7 @@ async function seed() {
 		await collections.settings.deleteMany({});
 		await collections.assistants.deleteMany({});
 		await collections.conversations.deleteMany({});
+		await collections.tools.deleteMany({});
 		console.log("Reset done");
 	}
 
@@ -239,6 +241,78 @@ async function seed() {
 		);
 		console.log("Done creating conversations.");
 	}
+
+	// generate Community Tools
+	if (flags.includes("tools") || flags.includes("all")) {
+		const tools = await Promise.all(
+			faker.helpers.multiple(
+				() => {
+					const _id = new ObjectId();
+					const displayName = faker.company.catchPhrase();
+					const description = faker.company.catchPhrase();
+					const color = faker.helpers.arrayElement([
+						"purple",
+						"blue",
+						"green",
+						"yellow",
+						"red",
+					]) satisfies ToolLogoColor;
+					const icon = faker.helpers.arrayElement([
+						"wikis",
+						"tools",
+						"camera",
+						"code",
+						"email",
+						"cloud",
+						"terminal",
+						"game",
+						"chat",
+						"speaker",
+						"video",
+					]) satisfies ToolLogoIcon;
+					const baseUrl = faker.helpers.arrayElement([
+						"stabilityai/stable-diffusion-3-medium",
+						"multimodalart/cosxl",
+						"gokaygokay/SD3-Long-Captioner",
+						"xichenhku/MimicBrush",
+					]);
+
+					// keep empty for populate for now
+
+					const user: User = faker.helpers.arrayElement(users);
+					const createdById = user._id;
+					const createdByName = user.username ?? user.name;
+
+					return {
+						type: "community" as const,
+						_id,
+						createdById,
+						createdByName,
+						displayName,
+						name: displayName.toLowerCase().replace(" ", "_"),
+						endpoint: "/test",
+						description,
+						color,
+						icon,
+						baseUrl,
+						inputs: [],
+						outputPath: null,
+						outputType: "str" as const,
+						showOutput: false,
+						useCount: faker.number.int({ min: 0, max: 100000 }),
+						last24HoursUseCount: faker.number.int({ min: 0, max: 1000 }),
+						createdAt: faker.date.recent({ days: 30 }),
+						updatedAt: faker.date.recent({ days: 30 }),
+						searchTokens: generateSearchTokens(displayName),
+						featured: faker.datatype.boolean(),
+					};
+				},
+				{ count: faker.number.int({ min: 10, max: 200 }) }
+			)
+		);
+
+		await collections.tools.insertMany(tools satisfies CommunityToolDB[]);
+	}
 }
 
 // run seed

src/lib/buildPrompt.ts

@@ -16,7 +16,11 @@ export async function buildPrompt({
 	tools,
 	toolResults,
 }: buildPromptOptions): Promise<string> {
-	const filteredMessages = messages.filter((m) => m.from !== "system");
+	const filteredMessages = messages;
+
+	if (filteredMessages[0].from === "system" && preprompt) {
+		filteredMessages[0].content = preprompt;
+	}
 
 	let prompt = model
 		.chatPromptRender({

src/lib/components/NavMenu.svelte

@@ -134,8 +134,17 @@
 			class="flex h-9 flex-none items-center gap-1.5 rounded-lg pl-2.5 pr-2 text-gray-500 hover:bg-gray-100 dark:text-gray-400 dark:hover:bg-gray-700"
 		>
 			Assistants
+		</a>
+	{/if}
+	<!-- XXX: feature_flag_tools -->
+	{#if $page.data.user?.isEarlyAccess}
+		<a
+			href="{base}/tools"
+			class="flex h-9 flex-none items-center gap-1.5 rounded-lg pl-2.5 pr-2 text-gray-500 hover:bg-gray-100 dark:text-gray-400 dark:hover:bg-gray-700"
+		>
+			Tools
 			<span
-				class="ml-auto rounded-full border border-gray-300 px-2 py-0.5 text-xs text-gray-500 dark:border-gray-500 dark:text-gray-400"
+				class="ml-auto rounded-full border border-purple-300 px-2 py-0.5 text-xs text-purple-500 dark:border-purple-500 dark:text-purple-400"
 				>New</span
 			>
 		</a>

src/lib/components/ToolLogo.svelte

@@ -0,0 +1,100 @@
+<script lang="ts">
+	import CarbonWikis from "~icons/carbon/wikis";
+	import CarbonTools from "~icons/carbon/tools";
+	import CarbonCamera from "~icons/carbon/camera";
+	import CarbonCode from "~icons/carbon/code";
+	import CarbonEmail from "~icons/carbon/email";
+	import CarbonCloud from "~icons/carbon/cloud-upload";
+	import CarbonTerminal from "~icons/carbon/terminal";
+	import CarbonGame from "~icons/carbon/game-console";
+	import CarbonChat from "~icons/carbon/chat-bot";
+	import CarbonSpeaker from "~icons/carbon/volume-up";
+	import CarbonVideo from "~icons/carbon/video";
+
+	export let color: string;
+	export let icon: string;
+	export let size: "md" | "lg" = "md";
+
+	$: gradientColor = (() => {
+		switch (color) {
+			case "purple":
+				return "#653789";
+			case "blue":
+				return "#375889";
+			case "green":
+				return "#37894E";
+			case "yellow":
+				return "#897C37";
+			case "red":
+				return "#893737";
+			default:
+				return "#FFF";
+		}
+	})();
+
+	let iconEl = CarbonWikis;
+
+	switch (icon) {
+		case "wikis":
+			iconEl = CarbonWikis;
+			break;
+		case "tools":
+			iconEl = CarbonTools;
+			break;
+		case "camera":
+			iconEl = CarbonCamera;
+			break;
+		case "code":
+			iconEl = CarbonCode;
+			break;
+		case "email":
+			iconEl = CarbonEmail;
+			break;
+		case "cloud":
+			iconEl = CarbonCloud;
+			break;
+		case "terminal":
+			iconEl = CarbonTerminal;
+			break;
+		case "game":
+			iconEl = CarbonGame;
+			break;
+		case "chat":
+			iconEl = CarbonChat;
+			break;
+		case "speaker":
+			iconEl = CarbonSpeaker;
+			break;
+		case "video":
+			iconEl = CarbonVideo;
+			break;
+	}
+
+	$: sizeClass = (() => {
+		switch (size) {
+			case "md":
+				return "size-14";
+			case "lg":
+				return "size-24";
+		}
+	})();
+</script>
+
+<div class="flex {sizeClass} items-center justify-center">
+	<svg xmlns="http://www.w3.org/2000/svg" class="absolute {sizeClass} h-full" viewBox="0 0 52 58">
+		<defs>
+			<linearGradient id="gradient-{gradientColor}" gradientTransform="rotate(90)">
+				<stop offset="0%" stop-color="#0E1523" />
+				<stop offset="100%" stop-color={gradientColor} />
+			</linearGradient>
+			<mask id="mask">
+				<path
+					d="M22.3043 1.2486C23.4279 0.603043 24.7025 0.263184 26 0.263184C27.2975 0.263184 28.5721 0.603043 29.6957 1.2486L48.3043 11.9373C49.4279 12.5828 50.361 13.5113 51.0097 14.6294C51.6584 15.7475 52 17.0158 52 18.3069V39.6902C52 40.9813 51.6584 42.2496 51.0097 43.3677C50.361 44.4858 49.4279 45.4143 48.3043 46.0598L29.6957 56.7514C28.5721 57.397 27.2975 57.7369 26 57.7369C24.7025 57.7369 23.4279 57.397 22.3043 56.7514L3.6957 46.0598C2.57209 45.4143 1.63904 44.4858 0.990308 43.3677C0.341578 42.2496 3.34785e-05 40.9813 5.18628e-07 39.6902V18.3099C-0.000485629 17.0183 0.340813 15.7494 0.989568 14.6307C1.63832 13.512 2.57166 12.5831 3.6957 11.9373L22.3043 1.2486Z"
+					fill="white"
+				/>
+			</mask>
+		</defs>
+		<rect width="100%" height="100%" fill="url(#gradient-{gradientColor})" mask="url(#mask)" />
+	</svg>
+	<svelte:component this={iconEl} class="relative {sizeClass} scale-50 text-clip text-gray-200" />
+</div>

src/lib/components/ToolsMenu.svelte

@@ -1,4 +1,5 @@
 <script lang="ts">
+	import { base } from "$app/paths";
 	import { page } from "$app/stores";
 	import { clickOutside } from "$lib/actions/clickOutside";
 	import { useSettingsStore } from "$lib/stores/settings";
@@ -14,15 +15,30 @@
 
 	// active tools are all the checked tools, either from settings or on by default
 	$: activeToolCount = $page.data.tools.filter(
-		(tool: ToolFront) => $settings?.tools?.[tool.name] ?? tool.isOnByDefault
+		(tool: ToolFront) =>
+			// community tools are always on by default
+			tool.type === "community" || $settings?.tools?.includes(tool._id)
 	).length;
 
-	function setAllTools(value: boolean) {
-		settings.instantSet({
-			tools: Object.fromEntries($page.data.tools.map((tool: ToolFront) => [tool.name, value])),
-		});
+	async function setAllTools(value: boolean) {
+		const configToolsIds = $page.data.tools
+			.filter((t: ToolFront) => t.type === "config")
+			.map((t: ToolFront) => t._id);
+
+		if (value) {
+			await settings.instantSet({
+				tools: Array.from(new Set([...configToolsIds, ...($settings?.tools ?? [])])),
+			});
+		} else {
+			await settings.instantSet({
+				tools: [],
+			});
+		}
 	}
+
 	$: allToolsEnabled = activeToolCount === $page.data.tools.length;
+
+	$: tools = $page.data.tools;
 </script>
 
 <details
@@ -67,24 +83,53 @@
 					{/if}
 				</button>
 			</div>
-			{#each $page.data.tools as tool}
-				{@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault}
+			<!-- XXX: feature_flag_tools -->
+			{#if $page.data.user?.isEarlyAccess}
+				<a
+					href="{base}/tools"
+					class="col-span-2 my-1 h-fit w-fit items-center justify-center rounded-full bg-purple-500/20 px-2.5 py-1.5 text-sm hover:bg-purple-500/30"
+				>
+					<span class="mr-1 rounded-full bg-purple-700 px-1.5 py-1 text-xs font-bold uppercase">
+						new
+					</span>
+					Browse community tools ({$page.data.communityToolCount ?? 0})
+				</a>
+			{/if}
+			{#each tools as tool}
+				{@const isChecked = $settings?.tools?.includes(tool._id)}
 				<div class="flex items-center gap-1.5">
-					<input
-						type="checkbox"
-						id={tool.name}
-						checked={isChecked}
-						disabled={loading}
-						on:click={async () => {
-							await settings.instantSet({
-								tools: {
-									...$settings.tools,
-									[tool.name]: !isChecked,
-								},
-							});
-						}}
-					/>
-					<label class="cursor-pointer" for={tool.name}>{tool.displayName ?? tool.name} </label>
+					{#if tool.type === "community"}
+						<input
+							type="checkbox"
+							id={tool._id}
+							checked={true}
+							class="rounded-xs font-semibold accent-purple-500 hover:accent-purple-600"
+							on:click|stopPropagation|preventDefault={async () => {
+								await settings.instantSet({
+									tools: $settings?.tools?.filter((t) => t !== tool._id) ?? [],
+								});
+							}}
+						/>
+					{:else}
+						<input
+							type="checkbox"
+							id={tool._id}
+							checked={isChecked}
+							disabled={loading}
+							on:click|stopPropagation={async () => {
+								if (isChecked) {
+									await settings.instantSet({
+										tools: ($settings?.tools ?? []).filter((t) => t !== tool._id),
+									});
+								} else {
+									await settings.instantSet({
+										tools: [...($settings?.tools ?? []), tool._id],
+									});
+								}
+							}}
+						/>
+					{/if}
+					<label class="cursor-pointer" for={tool._id}>{tool.displayName} </label>
 				</div>
 			{/each}
 		</div>

src/lib/components/chat/ChatWindow.svelte

@@ -155,8 +155,8 @@
 	const settings = useSettingsStore();
 
 	// active tools are all the checked tools, either from settings or on by default
-	$: activeTools = $page.data.tools.filter(
-		(tool: ToolFront) => $settings?.tools?.[tool.name] ?? tool.isOnByDefault
+	$: activeTools = $page.data.tools.filter((tool: ToolFront) =>
+		$settings?.tools?.includes(tool._id)
 	);
 	$: activeMimeTypes = [
 		...(!$page.data?.assistant && currentModel.tools

src/lib/components/chat/ToolUpdate.svelte

@@ -7,7 +7,6 @@
 	} from "$lib/utils/messageUpdates";
 
 	import CarbonTools from "~icons/carbon/tools";
-	import { toolHasName } from "$lib/utils/tools";
 	import type { ToolFront } from "$lib/types/Tool";
 	import { page } from "$app/stores";
 	import { onMount } from "svelte";
@@ -16,7 +15,7 @@
 	export let tool: MessageToolUpdate[];
 	export let loading: boolean = false;
 
-	const toolName = tool.find(isMessageToolCallUpdate)?.call.name;
+	const toolFnName = tool.find(isMessageToolCallUpdate)?.call.name;
 	$: toolError = tool.some(isMessageToolErrorUpdate);
 	$: toolDone = tool.some(isMessageToolResultUpdate);
 
@@ -26,12 +25,13 @@
 	let animation: Animation | undefined = undefined;
 
 	let isShowingLoadingBar = false;
+
 	onMount(() => {
 		if (!toolError && !toolDone && loading && loadingBarEl) {
 			loadingBarEl.classList.remove("hidden");
 			isShowingLoadingBar = true;
 			animation = loadingBarEl.animate([{ width: "0%" }, { width: "calc(100%+1rem)" }], {
-				duration: availableTools.find((tool) => tool.name === toolName)?.timeToUseMS,
+				duration: availableTools.find((tool) => tool.name === toolFnName)?.timeToUseMS,
 				fill: "forwards",
 			});
 		}
@@ -63,7 +63,7 @@
 		})();
 </script>
 
-{#if toolName && toolName !== "websearch"}
+{#if toolFnName && toolFnName !== "websearch"}
 	<details
 		class="group/tool my-2.5 w-fit cursor-pointer rounded-lg border border-gray-200 bg-white pl-1 pr-2.5 text-sm shadow-sm transition-all open:mb-3
         open:border-purple-500/10 open:bg-purple-600/5 open:shadow-sm dark:border-gray-800 dark:bg-gray-900 open:dark:border-purple-800/40 open:dark:bg-purple-800/10"
@@ -103,7 +103,8 @@
 			<span>
 				{toolError ? "Error calling" : toolDone ? "Called" : "Calling"} tool
 				<span class="font-semibold"
-					>{availableTools.find((el) => toolHasName(toolName, el))?.displayName}</span
+					>{availableTools.find((tool) => tool.name === toolFnName)?.displayName ??
+						toolFnName}</span
 				>
 			</span>
 		</summary>
@@ -115,10 +116,12 @@
 				</div>
 				<ul class="py-1 text-sm">
 					{#each Object.entries(toolUpdate.call.parameters ?? {}) as [k, v]}
-						<li>
-							<span class="font-semibold">{k}</span>:
-							<span>{v}</span>
-						</li>
+						{#if v !== null}
+							<li>
+								<span class="font-semibold">{k}</span>:
+								<span>{v}</span>
+							</li>
+						{/if}
 					{/each}
 				</ul>
 			{:else if toolUpdate.subtype === MessageToolUpdateType.Error}

src/lib/migrations/routines/03-add-tools-in-settings.ts

@@ -14,7 +14,7 @@ const addToolsToSettings: Migration = {
 			{
 				tools: { $exists: false },
 			},
-			{ $set: { tools: {} } }
+			{ $set: { tools: [] } }
 		);
 
 		settings

src/lib/migrations/routines/07-reset-tools-in-settings.ts

@@ -8,7 +8,7 @@ const resetTools: Migration = {
 	up: async () => {
 		const { settings } = collections;
 
-		await settings.updateMany({}, { $set: { tools: {} } });
+		await settings.updateMany({}, { $set: { tools: [] } });
 
 		return true;
 	},

src/lib/migrations/routines/08-reset-tools-in-settings-2.ts

@@ -0,0 +1,19 @@
+import type { Migration } from ".";
+import { collections } from "$lib/server/database";
+import { ObjectId } from "mongodb";
+
+const resetTools2: Migration = {
+	_id: new ObjectId("000000000008"),
+	name: "Reset tools to empty",
+	up: async () => {
+		const { settings } = collections;
+
+		await settings.updateMany({}, { $set: { tools: [] } });
+
+		return true;
+	},
+	runEveryTime: false,
+	runForHuggingChat: "only",
+};
+
+export default resetTools2;

src/lib/migrations/routines/index.ts

@@ -8,6 +8,7 @@ import updateMessageUpdates from "./04-update-message-updates";
 import updateMessageFiles from "./05-update-message-files";
 import trimMessageUpdates from "./06-trim-message-updates";
 import resetTools from "./07-reset-tools-in-settings";
+import resetTools2 from "./08-reset-tools-in-settings-2";
 
 export interface Migration {
 	_id: ObjectId;
@@ -27,4 +28,5 @@ export const migrations: Migration[] = [
 	updateMessageFiles,
 	trimMessageUpdates,
 	resetTools,
+	resetTools2,
 ];

src/lib/server/database.ts

@@ -13,6 +13,8 @@ import type { ConversationStats } from "$lib/types/ConversationStats";
 import type { MigrationResult } from "$lib/types/MigrationResult";
 import type { Semaphore } from "$lib/types/Semaphore";
 import type { AssistantStats } from "$lib/types/AssistantStats";
+import type { CommunityToolDB } from "$lib/types/Tool";
+
 import { logger } from "$lib/server/logger";
 import { building } from "$app/environment";
 import { onExit } from "./exitHandler";
@@ -83,6 +85,7 @@ export class Database {
 		const bucket = new GridFSBucket(db, { bucketName: "files" });
 		const migrationResults = db.collection<MigrationResult>("migrationResults");
 		const semaphores = db.collection<Semaphore>("semaphores");
+		const tools = db.collection<CommunityToolDB>("tools");
 
 		return {
 			conversations,
@@ -99,6 +102,7 @@ export class Database {
 			bucket,
 			migrationResults,
 			semaphores,
+			tools,
 		};
 	}
 
@@ -120,6 +124,7 @@ export class Database {
 			sessions,
 			messageEvents,
 			semaphores,
+			tools,
 		} = this.getCollections();
 
 		conversations
@@ -209,6 +214,10 @@ export class Database {
 		semaphores
 			.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 })
 			.catch((e) => logger.error(e));
+
+		tools.createIndex({ createdById: 1, userCount: -1 }).catch((e) => logger.error(e));
+		tools.createIndex({ userCount: 1 }).catch((e) => logger.error(e));
+		tools.createIndex({ last24HoursCount: 1 }).catch((e) => logger.error(e));
 	}
 }
 

src/lib/server/metrics.ts

@@ -3,7 +3,6 @@ import express from "express";
 import { logger } from "$lib/server/logger";
 import { env } from "$env/dynamic/private";
 import type { Model } from "$lib/types/Model";
-import type { Tool } from "$lib/types/Tool";
 import { onExit } from "./exitHandler";
 import { promisify } from "util";
 
@@ -26,9 +25,9 @@ interface Metrics {
 	};
 
 	tool: {
-		toolUseCount: Counter<Tool["name"]>;
-		toolUseCountError: Counter<Tool["name"]>;
-		toolUseDuration: Summary<Tool["name"]>;
+		toolUseCount: Counter<string>;
+		toolUseCountError: Counter<string>;
+		toolUseDuration: Summary<string>;
 		timeToChooseTools: Summary;
 	};
 }

src/lib/server/models.ts

@@ -12,7 +12,7 @@ import type { PreTrainedTokenizer } from "@xenova/transformers";
 import JSON5 from "json5";
 import { getTokenizer } from "$lib/utils/getTokenizer";
 import { logger } from "$lib/server/logger";
-import { ToolResultStatus } from "$lib/types/Tool";
+import { ToolResultStatus, type ToolInput } from "$lib/types/Tool";
 
 type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
 
@@ -96,14 +96,11 @@ async function getChatPromptRender(
 
 	const renderTemplate = ({ messages, preprompt, tools, toolResults }: ChatTemplateInput) => {
 		let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({
-			content:
-				message.files?.length && !tools?.length
-					? message.content + `\n This message has ${message.files.length} files attached`
-					: message.content,
+			content: message.content,
 			role: message.from,
 		}));
 
-		if (preprompt) {
+		if (preprompt && formattedMessages[0].role !== "system") {
 			formattedMessages = [
 				{
 					role: "system",
@@ -195,17 +192,41 @@ async function getChatPromptRender(
 			);
 		});
 
+		const mappedTools =
+			tools?.map((tool) => {
+				const inputs: Record<
+					string,
+					{
+						type: ToolInput["type"];
+						description: string;
+						required: boolean;
+					}
+				> = {};
+
+				for (const value of tool.inputs) {
+					if (value.paramType !== "fixed") {
+						inputs[value.name] = {
+							type: value.type,
+							description: value.description ?? "",
+							required: value.paramType === "required",
+						};
+					}
+				}
+
+				return {
+					name: tool.name,
+					description: tool.description,
+					parameter_definitions: inputs,
+				};
+			}) ?? [];
+
 		const output = tokenizer.apply_chat_template(formattedMessages, {
 			tokenize: false,
 			add_generation_prompt: true,
 			chat_template: chatTemplate,
 			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
 			// @ts-ignore
-			tools:
-				tools?.map(({ parameterDefinitions, ...tool }) => ({
-					parameter_definitions: parameterDefinitions,
-					...tool,
-				})) ?? [],
+			tools: mappedTools,
 			documents,
 		});
 

src/lib/server/textGeneration/index.ts

@@ -8,7 +8,7 @@ import {
 	getAssistantById,
 	processPreprompt,
 } from "./assistant";
-import { pickTools, runTools } from "./tools";
+import { filterToolsOnPreferences, runTools } from "./tools";
 import type { WebSearch } from "$lib/types/WebSearch";
 import {
 	type MessageUpdate,
@@ -20,7 +20,6 @@ import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
 import type { TextGenerationContext } from "./types";
 import type { ToolResult } from "$lib/types/Tool";
 import { toolHasName } from "../tools/utils";
-import directlyAnswer from "../tools/directlyAnswer";
 
 export async function* textGeneration(ctx: TextGenerationContext) {
 	yield* mergeAsyncGenerators([
@@ -63,8 +62,8 @@ async function* textGenerationWithoutTitle(
 	let toolResults: ToolResult[] = [];
 
 	if (model.tools && !conv.assistantId) {
-		const tools = pickTools(toolsPreference, Boolean(assistant));
-		const toolCallsRequired = tools.some((tool) => !toolHasName(directlyAnswer.name, tool));
+		const tools = await filterToolsOnPreferences(toolsPreference, Boolean(assistant));
+		const toolCallsRequired = tools.some((tool) => !toolHasName("directly_answer", tool));
 		if (toolCallsRequired) toolResults = yield* runTools(ctx, tools, preprompt);
 	}
 

src/lib/server/textGeneration/tools.ts

@@ -1,6 +1,6 @@
-import { ToolResultStatus, type ToolCall, type ToolResult } from "$lib/types/Tool";
+import { ToolResultStatus, type ToolCall, type Tool, type ToolResult } from "$lib/types/Tool";
 import { v4 as uuidV4 } from "uuid";
-import type { BackendTool, BackendToolContext } from "../tools";
+import { getCallMethod, toolFromConfigs, type BackendToolContext } from "../tools";
 import {
 	MessageToolUpdateType,
 	MessageUpdateStatus,
@@ -9,48 +9,44 @@ import {
 } from "$lib/types/MessageUpdate";
 import type { TextGenerationContext } from "./types";
 
-import { allTools } from "../tools";
 import directlyAnswer from "../tools/directlyAnswer";
 import websearch from "../tools/web/search";
 import { z } from "zod";
 import { logger } from "../logger";
 import { extractJson, toolHasName } from "../tools/utils";
-import type { MessageFile } from "$lib/types/Message";
 import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators";
 import { MetricsServer } from "../metrics";
 import { stringifyError } from "$lib/utils/stringifyError";
+import { collections } from "../database";
+import { ObjectId } from "mongodb";
+import type { Message } from "$lib/types/Message";
 
-function makeFilesPrompt(files: MessageFile[], fileMessageIndex: number): string {
-	if (files.length === 0) {
-		return "The user has not uploaded any files. Do not attempt to use any tools that require files";
-	}
-
-	const stringifiedFiles = files
-		.map(
-			(file, fileIndex) =>
-				`  - fileMessageIndex ${fileMessageIndex} | fileIndex ${fileIndex} | ${file.name} (${file.mime})`
-		)
-		.join("\n");
-	return `Attached ${files.length} file${files.length === 1 ? "" : "s"}:\n${stringifiedFiles}`;
-}
-
-export function pickTools(
-	toolsPreference: Record<string, boolean>,
+export async function filterToolsOnPreferences(
+	toolsPreference: Array<string>,
 	isAssistant: boolean
-): BackendTool[] {
+): Promise<Tool[]> {
 	// if it's an assistant, only support websearch for now
 	if (isAssistant) return [directlyAnswer, websearch];
 
 	// filter based on tool preferences, add the tools that are on by default
-	return allTools.filter((el) => {
+	const activeConfigTools = toolFromConfigs.filter((el) => {
 		if (el.isLocked && el.isOnByDefault) return true;
-		return toolsPreference?.[el.name] ?? el.isOnByDefault;
+		return toolsPreference?.includes(el._id.toString()) ?? el.isOnByDefault;
 	});
+
+	// find tool where the id is in toolsPreference
+	const activeCommunityTools = await collections.tools
+		.find({
+			_id: { $in: toolsPreference.map((el) => new ObjectId(el)) },
+		})
+		.toArray()
+		.then((el) => el.map((el) => ({ ...el, call: getCallMethod(el) })));
+	return [...activeConfigTools, ...activeCommunityTools];
 }
 
 async function* callTool(
 	ctx: BackendToolContext,
-	tools: BackendTool[],
+	tools: Tool[],
 	call: ToolCall
 ): AsyncGenerator<MessageUpdate, ToolResult | undefined, undefined> {
 	const uuid = uuidV4();
@@ -88,6 +84,8 @@ async function* callTool(
 			Date.now() - startTime
 		);
 
+		await collections.tools.findOneAndUpdate({ _id: tool._id }, { $inc: { useCount: 1 } });
+
 		return { ...toolResult, call } as ToolResult;
 	} catch (error) {
 		MetricsServer.getMetrics().tool.toolUseCountError.inc({ tool: call.name });
@@ -110,28 +108,75 @@ async function* callTool(
 
 export async function* runTools(
 	ctx: TextGenerationContext,
-	tools: BackendTool[],
+	tools: Tool[],
 	preprompt?: string
 ): AsyncGenerator<MessageUpdate, ToolResult[], undefined> {
 	const { endpoint, conv, messages, assistant, ip, username } = ctx;
 	const calls: ToolCall[] = [];
 
-	const messagesWithFilesPrompt = messages.map((message, idx) => {
-		if (!message.files?.length) return message;
+	const pickToolStartTime = Date.now();
+	// append a message with the list of all available files
+
+	const files = messages.reduce((acc, curr, idx) => {
+		if (curr.files) {
+			const prefix = (curr.from === "user" ? "input" : "ouput") + "_" + idx;
+			acc.push(
+				...curr.files.map(
+					(file, fileIdx) => `${prefix}_${fileIdx}.${file?.name?.split(".")?.pop()?.toLowerCase()}`
+				)
+			);
+		}
+		return acc;
+	}, [] as string[]);
+
+	let formattedMessages = messages.map((message, msgIdx) => {
+		let content = message.content;
+
+		if (message.files && message.files.length > 0) {
+			content +=
+				"\n\nAdded files: \n - " +
+				message.files
+					.map((file, fileIdx) => {
+						const prefix = message.from === "user" ? "input" : "output";
+						const fileName = file.name.split(".").pop()?.toLowerCase();
+
+						return `${prefix}_${msgIdx}_${fileIdx}.${fileName}`;
+					})
+					.join("\n - ");
+		}
+
 		return {
 			...message,
-			content: `${message.content}\n${makeFilesPrompt(message.files, idx)}`,
-		};
+			content,
+		} satisfies Message;
 	});
 
-	const pickToolStartTime = Date.now();
+	const fileMsg = {
+		id: crypto.randomUUID(),
+		from: "system",
+		content:
+			"Here is the list of available filenames that can be used as input for tools. Use the filenames that are in this list. \n The filename structure is as follows : {input for user|output for tool}_{message index in the conversation}_{file index in the list of files}.{file extension} \n - " +
+			files.join("\n - ") +
+			"\n\n\n",
+	} satisfies Message;
+
+	// put fileMsg before last if files.length > 0
+	formattedMessages = files.length
+		? [...formattedMessages.slice(0, -1), fileMsg, ...formattedMessages.slice(-1)]
+		: messages;
 
 	// do the function calling bits here
 	for await (const output of await endpoint({
-		messages: messagesWithFilesPrompt,
+		messages: formattedMessages,
 		preprompt,
 		generateSettings: assistant?.generateSettings,
-		tools,
+		tools: tools.map((tool) => ({
+			...tool,
+			inputs: tool.inputs.map((input) => ({
+				...input,
+				type: input.type === "file" ? "str" : input.type,
+			})),
+		})),
 	})) {
 		// model natively supports tool calls
 		if (output.token.toolCalls) {
@@ -146,7 +191,7 @@ export async function* runTools(
 				const rawCalls = await extractJson(output.generated_text);
 				const newCalls = rawCalls
 					.filter(isExternalToolCall)
-					.map(externalToToolCall)
+					.map((call) => externalToToolCall(call, tools))
 					.filter((call) => call !== undefined) as ToolCall[];
 
 				calls.push(...newCalls);
@@ -185,9 +230,10 @@ function isExternalToolCall(call: unknown): call is ExternalToolCall {
 	return externalToolCall.safeParse(call).success;
 }
 
-function externalToToolCall(call: ExternalToolCall): ToolCall | undefined {
+function externalToToolCall(call: ExternalToolCall, tools: Tool[]): ToolCall | undefined {
 	// Convert - to _ since some models insist on using _ instead of -
-	const tool = allTools.find((tool) => toolHasName(call.tool_name, tool));
+	const tool = tools.find((tool) => toolHasName(call.tool_name, tool));
+
 	if (!tool) {
 		logger.debug(`Model requested tool that does not exist: "${call.tool_name}". Skipping tool...`);
 		return;
@@ -195,23 +241,27 @@ function externalToToolCall(call: ExternalToolCall): ToolCall | undefined {
 
 	const parametersWithDefaults: Record<string, string> = {};
 
-	for (const [key, definition] of Object.entries(tool.parameterDefinitions)) {
-		const value = call.parameters[key];
+	for (const input of tool.inputs) {
+		const value = call.parameters[input.name];
 
 		// Required so ensure it's there, otherwise return undefined
-		if (definition.required) {
+		if (input.paramType === "required") {
 			if (value === undefined) {
 				logger.debug(
-					`Model requested tool "${call.tool_name}" but was missing required parameter "${key}". Skipping tool...`
+					`Model requested tool "${call.tool_name}" but was missing required parameter "${input.name}". Skipping tool...`
 				);
 				return;
 			}
-			parametersWithDefaults[key] = value;
+			parametersWithDefaults[input.name] = value;
 			continue;
 		}
 
 		// Optional so use default if not there
-		parametersWithDefaults[key] = value ?? definition.default;
+		parametersWithDefaults[input.name] = value;
+
+		if (input.paramType === "optional") {
+			parametersWithDefaults[input.name] ??= input.default.toString();
+		}
 	}
 
 	return {

src/lib/server/textGeneration/types.ts

@@ -12,7 +12,7 @@ export interface TextGenerationContext {
 	assistant?: Pick<Assistant, "rag" | "dynamicPrompt" | "generateSettings">;
 	isContinue: boolean;
 	webSearch: boolean;
-	toolsPreference: Record<string, boolean>;
+	toolsPreference: Array<string>;
 	promptedAt: Date;
 	ip: string;
 	username?: string;

src/lib/server/tools/calculator.ts

@@ -1,29 +1,38 @@
-import type { BackendTool } from ".";
+import type { ConfigTool } from "$lib/types/Tool";
+import { ObjectId } from "mongodb";
 import vm from "node:vm";
 
-const calculator: BackendTool = {
-	name: "query_calculator",
+const calculator: ConfigTool = {
+	_id: new ObjectId("00000000000000000000000C"),
+	type: "config",
+	description: "Calculate the result of a mathematical expression",
+	color: "blue",
+	icon: "code",
 	displayName: "Calculator",
-	description:
-		"A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, ** (power) and /, as well as parenthesis ().",
-	parameterDefinitions: {
-		equation: {
+	name: "calculator",
+	endpoint: null,
+	inputs: [
+		{
+			name: "equation",
+			type: "str",
 			description:
-				"The formula to evaluate. EXACTLY as you would plug into a calculator. No words, no letters, only numbers and operators. Letters will make the tool crash.",
-			type: "formula",
-			required: true,
+				"A mathematical expression to be evaluated. The result of the expression will be returned.",
+			paramType: "required",
 		},
-	},
-	async *call(params) {
+	],
+	outputComponent: null,
+	outputComponentIdx: null,
+	showOutput: false,
+	async *call({ equation }) {
 		try {
-			const blocks = String(params.equation).split("\n");
+			const blocks = String(equation).split("\n");
 			const query = blocks[blocks.length - 1].replace(/[^-()\d/*+.]/g, "");
 
 			return {
 				outputs: [{ calculator: `${query} = ${vm.runInNewContext(query)}` }],
 			};
 		} catch (cause) {
-			throw Error("Invalid expression", { cause });
+			throw new Error("Invalid expression", { cause });
 		}
 	},
 };

src/lib/server/tools/directlyAnswer.ts

@@ -1,12 +1,22 @@
-import type { BackendTool } from ".";
+import type { ConfigTool } from "$lib/types/Tool";
+import { ObjectId } from "mongodb";
 
-const directlyAnswer: BackendTool = {
-	name: "directly_answer",
+const directlyAnswer: ConfigTool = {
+	_id: new ObjectId("00000000000000000000000D"),
+	type: "config",
+	description: "Answer the user's query directly",
+	color: "blue",
+	icon: "chat",
+	displayName: "Directly Answer",
 	isOnByDefault: true,
-	isHidden: true,
 	isLocked: true,
-	description: "Use this tool to let the user know you wish to answer directly",
-	parameterDefinitions: {},
+	isHidden: true,
+	name: "directlyAnswer",
+	endpoint: null,
+	inputs: [],
+	outputComponent: null,
+	outputComponentIdx: null,
+	showOutput: false,
 	async *call() {
 		return {
 			outputs: [],

src/lib/server/tools/documentParser.ts

@@ -1,60 +0,0 @@
-import type { BackendTool } from ".";
-import { callSpace, getIpToken } from "./utils";
-import { downloadFile } from "$lib/server/files/downloadFile";
-
-type PdfParserInput = [Blob /* pdf */, string /* filename */];
-type PdfParserOutput = [string /* markdown */, Record<string, unknown> /* metadata */];
-
-const documentParser: BackendTool = {
-	name: "document_parser",
-	displayName: "Document Parser",
-	description: "Use this tool to parse any document and get its content in markdown format.",
-	mimeTypes: ["application/*", "text/*"],
-	parameterDefinitions: {
-		fileMessageIndex: {
-			description: "Index of the message containing the document file to parse",
-			type: "number",
-			required: true,
-		},
-		fileIndex: {
-			description: "Index of the document file to parse",
-			type: "number",
-			required: true,
-		},
-	},
-	async *call({ fileMessageIndex, fileIndex }, { conv, messages, ip, username }) {
-		fileMessageIndex = Number(fileMessageIndex);
-		fileIndex = Number(fileIndex);
-
-		const message = messages[fileMessageIndex];
-		const files = message?.files ?? [];
-		if (!files || files.length === 0) throw Error("User did not provide a pdf to parse");
-		if (fileIndex >= files.length) throw Error("Model provided an invalid file index");
-
-		const file = files[fileIndex];
-		const fileBlob = await downloadFile(files[fileIndex].value, conv._id)
-			.then((file) => fetch(`data:${file.mime};base64,${file.value}`))
-			.then((res) => res.blob());
-
-		const ipToken = await getIpToken(ip, username);
-
-		const outputs = await callSpace<PdfParserInput, PdfParserOutput>(
-			"huggingchat/document-parser",
-			"predict",
-			[fileBlob, file.name],
-			ipToken
-		);
-
-		let documentMarkdown = outputs[0];
-		// TODO: quick fix for avoiding context limit. eventually should use the tokenizer
-		if (documentMarkdown.length > 30_000) {
-			documentMarkdown = documentMarkdown.slice(0, 30_000) + "\n\n... (truncated)";
-		}
-		return {
-			outputs: [{ [file.name]: documentMarkdown }],
-			display: false,
-		};
-	},
-};
-
-export default documentParser;

src/lib/server/tools/images/editing.ts

@@ -1,95 +0,0 @@
-import type { BackendTool } from "..";
-import { uploadFile } from "../../files/uploadFile";
-import { MessageUpdateType } from "$lib/types/MessageUpdate";
-import { callSpace, getIpToken, type GradioImage } from "../utils";
-import { downloadFile } from "$lib/server/files/downloadFile";
-
-type ImageEditingInput = [
-	Blob /* image */,
-	string /* prompt */,
-	string /* negative prompt */,
-	number /* guidance scale */,
-	number /* steps */
-];
-type ImageEditingOutput = [GradioImage];
-
-const imageEditing: BackendTool = {
-	name: "image_editing",
-	displayName: "Image Editing",
-	description: "Use this tool to edit an image from a prompt.",
-	mimeTypes: ["image/*"],
-	parameterDefinitions: {
-		prompt: {
-			description:
-				"A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.",
-			type: "string",
-			required: true,
-		},
-		fileMessageIndex: {
-			description: "Index of the message containing the file to edit",
-			type: "number",
-			required: true,
-		},
-		fileIndex: {
-			description: "Index of the file to edit",
-			type: "number",
-			required: true,
-		},
-	},
-	async *call({ prompt, fileMessageIndex, fileIndex }, { conv, messages, ip, username }) {
-		prompt = String(prompt);
-		fileMessageIndex = Number(fileMessageIndex);
-		fileIndex = Number(fileIndex);
-
-		const message = messages[fileMessageIndex];
-		const images = message?.files ?? [];
-		if (!images || images.length === 0) throw Error("User did not provide an image to edit");
-		if (fileIndex >= images.length) throw Error("Model provided an invalid file index");
-		if (!images[fileIndex].mime.startsWith("image/")) {
-			throw Error("Model provided a file idex which is not an image");
-		}
-
-		// todo: should handle multiple images
-		const image = await downloadFile(images[fileIndex].value, conv._id)
-			.then((file) => fetch(`data:${file.mime};base64,${file.value}`))
-			.then((res) => res.blob());
-
-		const ipToken = await getIpToken(ip, username);
-
-		const outputs = await callSpace<ImageEditingInput, ImageEditingOutput>(
-			"multimodalart/cosxl",
-			"run_edit",
-			[
-				image,
-				prompt,
-				"", // negative prompt
-				7, // guidance scale
-				20, // steps
-			],
-			ipToken
-		);
-
-		const outputImage = await fetch(outputs[0].url)
-			.then((res) => res.blob())
-			.then((blob) => new File([blob], outputs[0].orig_name, { type: blob.type }))
-			.then((file) => uploadFile(file, conv));
-
-		yield {
-			type: MessageUpdateType.File,
-			name: outputImage.name,
-			sha: outputImage.value,
-			mime: outputImage.mime,
-		};
-
-		return {
-			outputs: [
-				{
-					imageEditing: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it. Be concise.`,
-				},
-			],
-			display: false,
-		};
-	},
-};
-
-export default imageEditing;

src/lib/server/tools/images/generation.ts

@@ -1,84 +0,0 @@
-import type { BackendTool } from "..";
-import { uploadFile } from "../../files/uploadFile";
-import { MessageUpdateType } from "$lib/types/MessageUpdate";
-import { callSpace, getIpToken, type GradioImage } from "../utils";
-
-type ImageGenerationInput = [string, string, number, boolean, number, number, number, number];
-type ImageGenerationOutput = [GradioImage, unknown];
-
-const imageGeneration: BackendTool = {
-	name: "image_generation",
-	displayName: "Image Generation",
-	description: "Use this tool to generate an image from a prompt.",
-	parameterDefinitions: {
-		prompt: {
-			description:
-				"A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.",
-			type: "string",
-			required: true,
-		},
-		negativePrompt: {
-			description:
-				"A prompt for things that should not be in the image. Simple terms, separate terms with a comma.",
-			type: "string",
-			required: false,
-			default: "",
-		},
-		width: {
-			description: "Width of the generated image.",
-			type: "number",
-			required: false,
-			default: 1024,
-		},
-		height: {
-			description: "Height of the generated image.",
-			type: "number",
-			required: false,
-			default: 1024,
-		},
-	},
-	async *call({ prompt, negativePrompt, width, height }, { conv, ip, username }) {
-		const ipToken = await getIpToken(ip, username);
-
-		const outputs = await callSpace<ImageGenerationInput, ImageGenerationOutput>(
-			"stabilityai/stable-diffusion-3-medium",
-			"/infer",
-			[
-				String(prompt), // prompt
-				String(negativePrompt), // negative prompt
-				Math.floor(Math.random() * 1000), // seed random
-				true, // randomize seed
-				Number(width), // number in 'Image Width' Number component
-				Number(height), // number in 'Image Height' Number component
-				5, // guidance scale
-				28, // steps
-			],
-			ipToken
-		);
-		const image = await fetch(outputs[0].url)
-			.then((res) => res.blob())
-			.then(
-				(blob) =>
-					new File([blob], `${prompt}.${blob.type.split("/")[1] ?? "png"}`, { type: blob.type })
-			)
-			.then((file) => uploadFile(file, conv));
-
-		yield {
-			type: MessageUpdateType.File,
-			name: image.name,
-			sha: image.value,
-			mime: image.mime,
-		};
-
-		return {
-			outputs: [
-				{
-					imageGeneration: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it. Be concise.`,
-				},
-			],
-			display: false,
-		};
-	},
-};
-
-export default imageGeneration;

src/lib/server/tools/index.ts

@@ -1,33 +1,294 @@
-import type { MessageUpdate } from "$lib/types/MessageUpdate";
-import type { Tool, ToolResultSuccess } from "$lib/types/Tool";
+import { MessageUpdateType } from "$lib/types/MessageUpdate";
+import {
+	ToolColor,
+	ToolIcon,
+	ToolOutputComponents,
+	type BackendCall,
+	type BaseTool,
+	type ConfigTool,
+	type ToolInput,
+} from "$lib/types/Tool";
+import type { TextGenerationContext } from "../textGeneration/types";
+
+import { z } from "zod";
+import JSON5 from "json5";
+import { env } from "$env/dynamic/private";
 
+import jp from "jsonpath";
 import calculator from "./calculator";
 import directlyAnswer from "./directlyAnswer";
-import imageEditing from "./images/editing";
-import imageGeneration from "./images/generation";
-import documentParser from "./documentParser";
 import fetchUrl from "./web/url";
 import websearch from "./web/search";
-import type { TextGenerationContext } from "../textGeneration/types";
+import { callSpace, getIpToken } from "./utils";
+import { uploadFile } from "../files/uploadFile";
+import type { MessageFile } from "$lib/types/Message";
+import { sha256 } from "$lib/utils/sha256";
+import { ObjectId } from "mongodb";
+import { isValidOutputComponent, ToolOutputPaths } from "./outputs";
+import { downloadFile } from "../files/downloadFile";
 
 export type BackendToolContext = Pick<
 	TextGenerationContext,
 	"conv" | "messages" | "assistant" | "ip" | "username"
 > & { preprompt?: string };
 
-export interface BackendTool extends Tool {
-	call(
-		params: Record<string, string | number | boolean>,
-		context: BackendToolContext
-	): AsyncGenerator<MessageUpdate, Omit<ToolResultSuccess, "status" | "call" | "type">, undefined>;
+const IOType = z.union([z.literal("str"), z.literal("int"), z.literal("float"), z.literal("bool")]);
+
+const toolInputBaseSchema = z.union([
+	z.object({
+		name: z.string().min(1).max(40),
+		description: z.string().max(100).optional(),
+		paramType: z.literal("required"),
+	}),
+	z.object({
+		name: z.string().min(1).max(40),
+		description: z.string().max(100).optional(),
+		paramType: z.literal("optional"),
+		default: z
+			.union([z.string().max(40), z.number(), z.boolean(), z.undefined()])
+			.transform((val) => (val === undefined ? "" : val)),
+	}),
+	z.object({
+		name: z.string().min(1).max(40),
+		paramType: z.literal("fixed"),
+		value: z
+			.union([z.string().max(40), z.number(), z.boolean(), z.undefined()])
+			.transform((val) => (val === undefined ? "" : val)),
+	}),
+]);
+
+const toolInputSchema = toolInputBaseSchema.and(
+	z.object({ type: IOType }).or(
+		z.object({
+			type: z.literal("file"),
+			mimeTypes: z.string().nonempty(),
+		})
+	)
+);
+
+export const editableToolSchema = z
+	.object({
+		name: z.string().min(1).max(40),
+		baseUrl: z.string().min(1).max(100),
+		endpoint: z.string().min(1).max(100),
+		inputs: z.array(toolInputSchema),
+		outputComponent: z.string().min(1).max(100),
+		showOutput: z.boolean(),
+		displayName: z.string().min(1).max(40),
+		color: ToolColor,
+		icon: ToolIcon,
+		description: z.string().min(1).max(100),
+	})
+	.transform((tool) => ({
+		...tool,
+		outputComponentIdx: parseInt(tool.outputComponent.split(";")[0]),
+		outputComponent: ToolOutputComponents.parse(tool.outputComponent.split(";")[1]),
+	}));
+
+export const configTools = z
+	.array(
+		z
+			.object({
+				name: z.string(),
+				description: z.string(),
+				endpoint: z.union([z.string(), z.null()]),
+				inputs: z.array(toolInputSchema),
+				outputComponent: ToolOutputComponents.or(z.null()),
+				outputComponentIdx: z.number().int().default(0),
+				showOutput: z.boolean(),
+				_id: z
+					.string()
+					.length(24)
+					.regex(/^[0-9a-fA-F]{24}$/)
+					.transform((val) => new ObjectId(val)),
+				baseUrl: z.string().optional(),
+				displayName: z.string(),
+				color: ToolColor,
+				icon: ToolIcon,
+				isOnByDefault: z.optional(z.literal(true)),
+				isLocked: z.optional(z.literal(true)),
+				isHidden: z.optional(z.literal(true)),
+			})
+			.transform((val) => ({
+				type: "config" as const,
+				...val,
+				call: getCallMethod(val),
+			}))
+	)
+	// add the extra hardcoded tools
+	.transform((val) => [...val, calculator, directlyAnswer, fetchUrl, websearch]);
+
+export function getCallMethod(tool: Omit<BaseTool, "call">): BackendCall {
+	return async function* (params, ctx) {
+		if (
+			tool.endpoint === null ||
+			!tool.baseUrl ||
+			!tool.outputComponent ||
+			tool.outputComponentIdx === null
+		) {
+			throw new Error(`Tool function ${tool.name} has no endpoint`);
+		}
+
+		const ipToken = await getIpToken(ctx.ip, ctx.username);
+
+		function coerceInput(value: unknown, type: ToolInput["type"]) {
+			const valueStr = String(value);
+			switch (type) {
+				case "str":
+					return valueStr;
+				case "int":
+					return parseInt(valueStr);
+				case "float":
+					return parseFloat(valueStr);
+				case "bool":
+					return valueStr === "true";
+				default:
+					throw new Error(`Unsupported type ${type}`);
+			}
+		}
+		const inputs = tool.inputs.map(async (input) => {
+			if (input.type === "file" && input.paramType !== "required") {
+				throw new Error("File inputs are always required and cannot be optional or fixed");
+			}
+
+			if (input.paramType === "fixed") {
+				return coerceInput(input.value, input.type);
+			} else if (input.paramType === "optional") {
+				return coerceInput(params[input.name] ?? input.default, input.type);
+			} else if (input.paramType === "required") {
+				if (params[input.name] === undefined) {
+					throw new Error(`Missing required input ${input.name}`);
+				}
+
+				if (input.type === "file") {
+					// todo: parse file here !
+					// structure is {input|output}-{msgIdx}-{fileIdx}-{filename}
+
+					const filename = params[input.name];
+
+					if (!filename || typeof filename !== "string") {
+						throw new Error(`Filename is not a string`);
+					}
+
+					const messages = ctx.messages;
+
+					const msgIdx = parseInt(filename.split("_")[1]);
+					const fileIdx = parseInt(filename.split("_")[2]);
+
+					if (Number.isNaN(msgIdx) || Number.isNaN(fileIdx)) {
+						throw Error(`Message index or file index is missing`);
+					}
+
+					if (msgIdx >= messages.length) {
+						throw Error(`Message index ${msgIdx} is out of bounds`);
+					}
+
+					const file = messages[msgIdx].files?.[fileIdx];
+
+					if (!file) {
+						throw Error(`File index ${fileIdx} is out of bounds`);
+					}
+
+					const blob = await downloadFile(file.value, ctx.conv._id)
+						.then((file) => fetch(`data:${file.mime};base64,${file.value}`))
+						.then((res) => res.blob())
+						.catch((err) => {
+							throw Error("Failed to download file", { cause: err });
+						});
+
+					return blob;
+				} else {
+					return coerceInput(params[input.name], input.type);
+				}
+			}
+		});
+
+		const outputs = await callSpace(
+			tool.baseUrl,
+			tool.endpoint,
+			await Promise.all(inputs),
+			ipToken
+		);
+
+		if (!isValidOutputComponent(tool.outputComponent)) {
+			throw new Error(`Tool output component is not defined`);
+		}
+
+		const { type, path } = ToolOutputPaths[tool.outputComponent];
+
+		if (!path || !type) {
+			throw new Error(`Tool output type ${tool.outputComponent} is not supported`);
+		}
+
+		const files: MessageFile[] = [];
+
+		const toolOutputs: Array<Record<string, string>> = [];
+
+		if (outputs.length <= tool.outputComponentIdx) {
+			throw new Error(`Tool output component index is out of bounds`);
+		}
+
+		// if its not an object, return directly
+		if (
+			outputs[tool.outputComponentIdx] !== undefined &&
+			typeof outputs[tool.outputComponentIdx] !== "object"
+		) {
+			return { outputs: [{ [tool.name + "-0"]: outputs[tool.outputComponentIdx] }] };
+		}
+
+		await Promise.all(
+			jp
+				.query(outputs[tool.outputComponentIdx], path)
+				.map(async (output: string | string[], idx) => {
+					const arrayedOutput = Array.isArray(output) ? output : [output];
+					if (type === "file") {
+						// output files are actually URLs
+
+						await Promise.all(
+							arrayedOutput.map(async (output, idx) => {
+								await fetch(output)
+									.then((res) => res.blob())
+									.then(async (blob) => {
+										const mimeType = blob.type;
+										const fileType = blob.type.split("/")[1] ?? mimeType?.split("/")[1];
+										return new File(
+											[blob],
+											`${idx}-${await sha256(JSON.stringify(params))}.${fileType}`,
+											{
+												type: fileType,
+											}
+										);
+									})
+									.then((file) => uploadFile(file, ctx.conv))
+									.then((file) => files.push(file));
+							})
+						);
+
+						toolOutputs.push({
+							[tool.name + "-" + idx.toString()]:
+								"A file has been generated. Answer as if the user can already see the file. Do not try to insert the file. The user can already see the file. Do not try to describe the file as the model cannot interact with it. Be concise.",
+						});
+					} else {
+						for (const output of arrayedOutput) {
+							toolOutputs.push({
+								[tool.name + "-" + idx.toString()]: output,
+							});
+						}
+					}
+				})
+		);
+
+		for (const file of files) {
+			yield {
+				type: MessageUpdateType.File,
+				name: file.name,
+				sha: file.value,
+				mime: file.mime,
+			};
+		}
+
+		return { outputs: toolOutputs, display: tool.showOutput };
+	};
 }
 
-export const allTools: BackendTool[] = [
-	directlyAnswer,
-	websearch,
-	imageGeneration,
-	fetchUrl,
-	imageEditing,
-	documentParser,
-	calculator,
-];
+export const toolFromConfigs = configTools.parse(JSON5.parse(env.TOOLS)) satisfies ConfigTool[];

src/lib/server/tools/outputs.ts

@@ -0,0 +1,32 @@
+import type { ToolIOType, ToolOutputComponents } from "$lib/types/Tool";
+
+export const ToolOutputPaths: Record<
+	ToolOutputComponents,
+	{
+		type: ToolIOType;
+		path: string;
+	}
+> = {
+	textbox: {
+		type: "str",
+		path: "$",
+	},
+	markdown: {
+		type: "str",
+		path: "$",
+	},
+	image: {
+		type: "file",
+		path: "$.url",
+	},
+	gallery: {
+		type: "file",
+		path: "$[*].image.url",
+	},
+};
+
+export const isValidOutputComponent = (
+	outputComponent: string
+): outputComponent is keyof typeof ToolOutputPaths => {
+	return Object.keys(ToolOutputPaths).includes(outputComponent);
+};

src/lib/server/tools/utils.ts

@@ -1,6 +1,7 @@
 import { env } from "$env/dynamic/private";
 import { Client } from "@gradio/client";
 import { SignJWT } from "jose";
+import { logger } from "../logger";
 import JSON5 from "json5";
 
 export type GradioImage = {
@@ -31,13 +32,17 @@ export async function callSpace<TInput extends unknown[], TOutput extends unknow
 			return super.fetch(input, init);
 		}
 	}
-
 	const client = await CustomClient.connect(name, {
 		hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`,
 	});
+
 	return await client
 		.predict(func, parameters)
-		.then((res) => (res as unknown as GradioResponse).data as TOutput);
+		.then((res) => (res as unknown as GradioResponse).data as TOutput)
+		.catch((e) => {
+			logger.error(e);
+			throw e;
+		});
 }
 
 export async function getIpToken(ip: string, username?: string) {

src/lib/server/tools/web/search.ts

@@ -1,19 +1,28 @@
-import type { BackendTool } from "..";
+import type { ConfigTool } from "$lib/types/Tool";
+import { ObjectId } from "mongodb";
 import { runWebSearch } from "../../websearch/runWebSearch";
 
-const websearch: BackendTool = {
-	name: "websearch",
+const websearch: ConfigTool = {
+	_id: new ObjectId("00000000000000000000000A"),
+	type: "config",
+	description: "Search the web for answers to the user's query",
+	color: "blue",
+	icon: "wikis",
 	displayName: "Web Search",
-	description:
-		"Use this tool to search web pages for answers that will help answer the user's query. Only use this tool if you need specific resources from the internet.",
-	parameterDefinitions: {
-		query: {
-			required: true,
-			type: "string",
+	name: "websearch",
+	endpoint: null,
+	inputs: [
+		{
+			name: "query",
+			type: "str",
 			description:
 				"A search query which will be used to fetch the most relevant snippets regarding the user's query",
+			paramType: "required",
 		},
-	},
+	],
+	outputComponent: null,
+	outputComponentIdx: null,
+	showOutput: false,
 	async *call({ query }, { conv, assistant, messages }) {
 		const webSearchToolResults = yield* runWebSearch(conv, messages, assistant?.rag, String(query));
 		const chunks = webSearchToolResults?.contextSources

src/lib/server/tools/web/url.ts

@@ -1,23 +1,33 @@
 import { stringifyMarkdownElementTree } from "$lib/server/websearch/markdown/utils/stringify";
 import { scrapeUrl } from "$lib/server/websearch/scrape/scrape";
-import type { BackendTool } from "..";
+import type { ConfigTool } from "$lib/types/Tool";
+import { ObjectId } from "mongodb";
 
-const fetchUrl: BackendTool = {
-	name: "fetch_url",
-	displayName: "URL Fetcher",
-	description: "A tool that can be used to fetch an URL and return the content directly.",
-	parameterDefinitions: {
-		url: {
-			description: "The url that should be fetched.",
+const fetchUrl: ConfigTool = {
+	_id: new ObjectId("00000000000000000000000B"),
+	type: "config",
+	description: "Fetch the contents of a URL",
+	color: "blue",
+	icon: "cloud",
+	displayName: "Fetch URL",
+	name: "fetchUrl",
+	endpoint: null,
+	inputs: [
+		{
+			name: "url",
 			type: "str",
-			required: true,
+			description: "The URL of the webpage to fetch",
+			paramType: "required",
 		},
-	},
-	async *call(params) {
-		const blocks = String(params.url).split("\n");
-		const url = blocks[blocks.length - 1];
+	],
+	outputComponent: null,
+	outputComponentIdx: null,
+	showOutput: false,
+	async *call({ url }) {
+		const blocks = String(url).split("\n");
+		const urlStr = blocks[blocks.length - 1];
 
-		const { title, markdownTree } = await scrapeUrl(url, Infinity);
+		const { title, markdownTree } = await scrapeUrl(urlStr, Infinity);
 
 		return {
 			outputs: [{ title, text: stringifyMarkdownElementTree(markdownTree) }],

src/lib/server/usageLimits.ts

@@ -17,6 +17,7 @@ export const usageLimitsSchema = z
 				return val;
 			}, z.coerce.number().optional())
 			.optional(), // how many messages per minute
+		tools: z.coerce.number().optional(), // how many tools
 	})
 	.optional();
 

src/lib/stores/settings.ts

@@ -15,7 +15,7 @@ type SettingsStore = {
 	customPrompts: Record<string, string>;
 	recentlySaved: boolean;
 	assistants: Array<ObjectId | string>;
-	tools?: Record<string, boolean>;
+	tools?: Array<string>;
 	disableStream: boolean;
 };
 

src/lib/types/Report.ts

@@ -6,6 +6,7 @@ import type { Timestamps } from "./Timestamps";
 export interface Report extends Timestamps {
 	_id: ObjectId;
 	createdBy: User["_id"] | string;
-	assistantId: Assistant["_id"];
+	object: "assistant" | "tool";
+	contentId: Assistant["_id"];
 	reason?: string;
 }

src/lib/types/Settings.ts

@@ -21,7 +21,7 @@ export interface Settings extends Timestamps {
 	customPrompts?: Record<string, string>;
 
 	assistants?: Assistant["_id"][];
-	tools?: Record<string, boolean>;
+	tools?: string[];
 	disableStream: boolean;
 }
 
@@ -33,6 +33,6 @@ export const DEFAULT_SETTINGS = {
 	hideEmojiOnSidebar: false,
 	customPrompts: {},
 	assistants: [],
-	tools: {},
+	tools: [],
 	disableStream: false,
 } satisfies SettingsEditable;

src/lib/types/Tool.ts

@@ -1,33 +1,147 @@
-type ToolInput =
-	| {
-			description: string;
-			type: string;
-			required: true;
-	  }
-	| {
-			description: string;
-			type: string;
-			required: false;
-			default: string | number | boolean;
-	  };
-
-export interface Tool {
+import type { ObjectId } from "mongodb";
+import type { User } from "./User";
+import type { Timestamps } from "./Timestamps";
+import type { BackendToolContext } from "$lib/server/tools";
+import type { MessageUpdate } from "./MessageUpdate";
+import { z } from "zod";
+
+export const ToolColor = z.union([
+	z.literal("purple"),
+	z.literal("blue"),
+	z.literal("green"),
+	z.literal("yellow"),
+	z.literal("red"),
+]);
+
+export const ToolIcon = z.union([
+	z.literal("wikis"),
+	z.literal("tools"),
+	z.literal("camera"),
+	z.literal("code"),
+	z.literal("email"),
+	z.literal("cloud"),
+	z.literal("terminal"),
+	z.literal("game"),
+	z.literal("chat"),
+	z.literal("speaker"),
+	z.literal("video"),
+]);
+
+export const ToolOutputComponents = z
+	.string()
+	.toLowerCase()
+	.pipe(
+		z.union([z.literal("textbox"), z.literal("markdown"), z.literal("image"), z.literal("gallery")])
+	);
+
+export type ToolOutputComponents = z.infer<typeof ToolOutputComponents>;
+
+export type ToolLogoColor = z.infer<typeof ToolColor>;
+export type ToolLogoIcon = z.infer<typeof ToolIcon>;
+
+export type ToolIOType = "str" | "int" | "float" | "bool" | "file";
+
+export type ToolInputRequired = {
+	paramType: "required";
+	name: string;
+	description?: string;
+};
+
+export type ToolInputOptional = {
+	paramType: "optional";
+	name: string;
+	description?: string;
+	default: string | number | boolean;
+};
+
+export type ToolInputFixed = {
+	paramType: "fixed";
 	name: string;
-	displayName?: string;
+	value: string | number | boolean;
+};
+
+type ToolInputBase = ToolInputRequired | ToolInputOptional | ToolInputFixed;
+
+export type ToolInputFile = ToolInputBase & {
+	type: "file";
+	mimeTypes: string;
+};
+
+export type ToolInputSimple = ToolInputBase & {
+	type: Exclude<ToolIOType, "file">;
+};
+
+export type ToolInput = ToolInputFile | ToolInputSimple;
+
+export interface BaseTool {
+	_id: ObjectId;
+
+	name: string; // name that will be shown to the AI
+
+	baseUrl?: string; // namespace for the tool
+	endpoint: string | null; // endpoint to call in gradio, if null we expect to override this function in code
+	outputComponent: string | null; // Gradio component type to use for the output
+	outputComponentIdx: number | null; // index of the output component
+
+	inputs: Array<ToolInput>;
+	showOutput: boolean; // show output in chat or not
+
+	call: BackendCall;
+
+	// for displaying in the UI
+	displayName: string;
+	color: ToolLogoColor;
+	icon: ToolLogoIcon;
 	description: string;
-	/** List of mime types that tool accepts */
-	mimeTypes?: string[];
-	parameterDefinitions: Record<string, ToolInput>;
-	spec?: string;
-	isOnByDefault?: true; // will it be toggled if the user hasn't tweaked it in settings ?
-	isLocked?: true; // can the user enable/disable it ?
-	isHidden?: true; // should it be hidden from the user ?
 }
 
-export type ToolFront = Pick<
-	Tool,
-	"name" | "displayName" | "description" | "isOnByDefault" | "isLocked" | "mimeTypes"
-> & { timeToUseMS?: number };
+export interface ConfigTool extends BaseTool {
+	type: "config";
+	isOnByDefault?: true;
+	isLocked?: true;
+	isHidden?: true;
+}
+
+export interface CommunityTool extends BaseTool, Timestamps {
+	type: "community";
+
+	createdById: User["_id"] | string; // user id or session
+	createdByName?: User["username"];
+
+	// used to compute popular & trending
+	useCount: number;
+	last24HoursUseCount: number;
+
+	featured: boolean;
+	searchTokens: string[];
+}
+
+// no call function in db
+export type CommunityToolDB = Omit<CommunityTool, "call">;
+
+export type CommunityToolEditable = Omit<
+	CommunityToolDB,
+	| "_id"
+	| "useCount"
+	| "last24HoursUseCount"
+	| "createdById"
+	| "createdByName"
+	| "featured"
+	| "searchTokens"
+	| "type"
+	| "createdAt"
+	| "updatedAt"
+>;
+
+export type Tool = ConfigTool | CommunityTool;
+
+export type ToolFront = Pick<Tool, "type" | "name" | "displayName" | "description"> & {
+	_id: string;
+	isOnByDefault: boolean;
+	isLocked: boolean;
+	mimeTypes: string[];
+	timeToUseMS?: number;
+};
 
 export enum ToolResultStatus {
 	Success = "success",
@@ -51,3 +165,8 @@ export interface ToolCall {
 	name: string;
 	parameters: Record<string, string | number | boolean>;
 }
+
+export type BackendCall = (
+	params: Record<string, string | number | boolean>,
+	context: BackendToolContext
+) => AsyncGenerator<MessageUpdate, Omit<ToolResultSuccess, "status" | "call" | "type">, undefined>;

src/lib/utils/getGradioApi.ts

@@ -0,0 +1,11 @@
+import { base } from "$app/paths";
+import type { Client } from "@gradio/client";
+
+export type ApiReturnType = Awaited<ReturnType<typeof Client.prototype.view_api>>;
+
+export async function getGradioApi(space: string) {
+	const api: ApiReturnType = await fetch(`${base}/api/spaces-config?space=${space}`).then((res) =>
+		res.json()
+	);
+	return api;
+}

src/lib/utils/messageUpdates.ts

@@ -48,7 +48,7 @@ type MessageUpdateRequestOptions = {
 	isRetry: boolean;
 	isContinue: boolean;
 	webSearch: boolean;
-	tools?: Record<string, boolean>;
+	tools?: Array<string>;
 	files?: MessageFile[];
 };
 export async function fetchMessageUpdates(

src/lib/utils/tools.ts

@@ -7,3 +7,19 @@ import type { Tool } from "$lib/types/Tool";
 export function toolHasName(name: string, tool: Pick<Tool, "name">): boolean {
 	return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_");
 }
+
+export const colors = ["purple", "blue", "green", "yellow", "red"] as const;
+
+export const icons = [
+	"wikis",
+	"tools",
+	"camera",
+	"code",
+	"email",
+	"cloud",
+	"terminal",
+	"game",
+	"chat",
+	"speaker",
+	"video",
+] as const;

src/routes/+layout.server.ts

@@ -8,8 +8,9 @@ import { DEFAULT_SETTINGS } from "$lib/types/Settings";
 import { env } from "$env/dynamic/private";
 import { ObjectId } from "mongodb";
 import type { ConvSidebar } from "$lib/types/ConvSidebar";
-import { allTools } from "$lib/server/tools";
+import { toolFromConfigs } from "$lib/server/tools";
 import { MetricsServer } from "$lib/server/metrics";
+import type { ToolFront, ToolInputFile } from "$lib/types/Tool";
 
 export const load: LayoutServerLoad = async ({ locals, depends, request }) => {
 	depends(UrlDependency.ConversationList);
@@ -107,6 +108,25 @@ export const load: LayoutServerLoad = async ({ locals, depends, request }) => {
 	}
 
 	const toolUseDuration = (await MetricsServer.getMetrics().tool.toolUseDuration.get()).values;
+
+	const configToolIds = toolFromConfigs.map((el) => el._id.toString());
+
+	const activeCommunityToolIds = (settings?.tools ?? []).filter(
+		(key) => !configToolIds.includes(key)
+	);
+
+	const communityTools = await collections.tools
+		.find({ _id: { $in: activeCommunityToolIds.map((el) => new ObjectId(el)) } })
+		.toArray()
+		.then((tools) =>
+			tools.map((tool) => ({
+				...tool,
+				isHidden: false,
+				isOnByDefault: true,
+				isLocked: true,
+			}))
+		);
+
 	return {
 		conversations: conversations.map((conv) => {
 			if (settings?.hideEmojiOnSidebar) {
@@ -146,7 +166,11 @@ export const load: LayoutServerLoad = async ({ locals, depends, request }) => {
 				DEFAULT_SETTINGS.shareConversationsWithModelAuthors,
 			customPrompts: settings?.customPrompts ?? {},
 			assistants: userAssistants,
-			tools: settings?.tools ?? {},
+			tools:
+				settings?.tools ??
+				toolFromConfigs
+					.filter((el) => !el.isHidden && el.isOnByDefault)
+					.map((el) => el._id.toString()),
 			disableStream: settings?.disableStream ?? DEFAULT_SETTINGS.disableStream,
 		},
 		models: models.map((model) => ({
@@ -171,19 +195,29 @@ export const load: LayoutServerLoad = async ({ locals, depends, request }) => {
 			unlisted: model.unlisted,
 		})),
 		oldModels,
-		tools: allTools
-			.filter((tool) => !tool.isHidden)
-			.map((tool) => ({
-				name: tool.name,
-				displayName: tool.displayName,
-				description: tool.description,
-				mimeTypes: tool.mimeTypes,
-				isOnByDefault: tool.isOnByDefault,
-				isLocked: tool.isLocked,
-				timeToUseMS:
-					toolUseDuration.find((el) => el.labels.tool === tool.name && el.labels.quantile === 0.9)
-						?.value ?? 15_000,
-			})),
+		tools: [...toolFromConfigs, ...communityTools]
+			.filter((tool) => !tool?.isHidden)
+			.map(
+				(tool) =>
+					({
+						_id: tool._id.toString(),
+						type: tool.type,
+						displayName: tool.displayName,
+						name: tool.name,
+						description: tool.description,
+						mimeTypes: (tool.inputs ?? [])
+							.filter((input): input is ToolInputFile => input.type === "file")
+							.map((input) => (input as ToolInputFile).mimeTypes)
+							.flat(),
+						isOnByDefault: tool.isOnByDefault ?? true,
+						isLocked: tool.isLocked ?? true,
+						timeToUseMS:
+							toolUseDuration.find(
+								(el) => el.labels.tool === tool._id.toString() && el.labels.quantile === 0.9
+							)?.value ?? 15_000,
+					} satisfies ToolFront)
+			),
+		communityToolCount: await collections.tools.countDocuments({ type: "community" }),
 		assistants: assistants
 			.filter((el) => userAssistantsSet.has(el._id.toString()))
 			.map((el) => ({

src/routes/api/spaces-config/+server.ts

@@ -0,0 +1,31 @@
+import { Client } from "@gradio/client";
+
+export async function GET({ url, locals }) {
+	// XXX: feature_flag_tools
+	if (!locals.user?.isEarlyAccess) {
+		return new Response("Not early access", { status: 403 });
+	}
+
+	const space = url.searchParams.get("space");
+
+	if (!space) {
+		return new Response("Missing space", { status: 400 });
+	}
+
+	try {
+		const api = await (await Client.connect(space)).view_api();
+		return new Response(JSON.stringify(api), {
+			status: 200,
+			headers: {
+				"Content-Type": "application/json",
+			},
+		});
+	} catch (e) {
+		return new Response(JSON.stringify({ error: true, message: "Failed to get space API" }), {
+			status: 400,
+			headers: {
+				"Content-Type": "application/json",
+			},
+		});
+	}
+}

src/routes/assistants/+page.server.ts

@@ -55,8 +55,8 @@ export const load = async ({ url, locals }) => {
 		.assistants.find(filter)
 		.skip(NUM_PER_PAGE * pageIndex)
 		.sort({
-			...(sort === SortKey.TRENDING && { last24HoursCount: -1 }),
-			userCount: -1,
+			...(sort === SortKey.TRENDING && { last24HoursUseCount: -1 }),
+			useCount: -1,
 		})
 		.limit(NUM_PER_PAGE)
 		.toArray();

src/routes/conversation/[id]/+server.ts

@@ -159,12 +159,23 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			is_continue: z.optional(z.boolean()),
 			web_search: z.optional(z.boolean()),
 			tools: z
-				.record(z.boolean())
+				.array(z.string())
 				.optional()
 				.transform((tools) =>
 					// disable tools on huggingchat android app
-					request.headers.get("user-agent")?.includes("co.huggingface.chat_ui_android") ? {} : tools
+					request.headers.get("user-agent")?.includes("co.huggingface.chat_ui_android") ? [] : tools
 				),
+
+			files: z.optional(
+				z.array(
+					z.object({
+						type: z.literal("base64").or(z.literal("hash")),
+						name: z.string(),
+						value: z.string(),
+						mime: z.string(),
+					})
+				)
+			),
 		})
 		.parse(JSON.parse(json));
 
@@ -419,7 +430,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 					assistant: undefined,
 					isContinue: isContinue ?? false,
 					webSearch: webSearch ?? false,
-					toolsPreference: toolsPreferences ?? {},
+					toolsPreference: toolsPreferences ?? [],
 					promptedAt,
 					ip: getClientAddress(),
 					username: locals.user?.username,

src/routes/settings/(nav)/+server.ts

@@ -2,6 +2,7 @@ import { collections } from "$lib/server/database";
 import { z } from "zod";
 import { authCondition } from "$lib/server/auth";
 import { DEFAULT_SETTINGS, type SettingsEditable } from "$lib/types/Settings";
+import { toolFromConfigs } from "$lib/server/tools/index.js";
 
 export async function POST({ request, locals }) {
 	const body = await request.json();
@@ -15,11 +16,19 @@ export async function POST({ request, locals }) {
 			ethicsModalAccepted: z.boolean().optional(),
 			activeModel: z.string().default(DEFAULT_SETTINGS.activeModel),
 			customPrompts: z.record(z.string()).default({}),
-			tools: z.record(z.boolean()).optional(),
+			tools: z.array(z.string()).optional(),
 			disableStream: z.boolean().default(false),
 		})
 		.parse(body) satisfies SettingsEditable;
 
+	// only allow tools to be set to community tools if user is early access
+	// XXX: feature_flag_tools
+	if (!locals.user?.isEarlyAccess) {
+		settings.tools = settings.tools?.filter((toolId) => {
+			return toolFromConfigs.some((tool) => tool._id.toString() === toolId);
+		});
+	}
+
 	await collections.settings.updateOne(
 		authCondition(locals),
 		{

src/routes/settings/(nav)/assistants/[assistantId]/+page.server.ts

@@ -63,7 +63,8 @@ export const actions: Actions = {
 		// is there already a report from this user for this model ?
 		const report = await collections.reports.findOne({
 			createdBy: locals.user?._id ?? locals.sessionId,
-			assistantId: new ObjectId(params.assistantId),
+			object: "assistant",
+			contentId: new ObjectId(params.assistantId),
 		});
 
 		if (report) {
@@ -79,7 +80,8 @@ export const actions: Actions = {
 
 		const { acknowledged } = await collections.reports.insertOne({
 			_id: new ObjectId(),
-			assistantId: new ObjectId(params.assistantId),
+			contentId: new ObjectId(params.assistantId),
+			object: "assistant",
 			createdBy: locals.user?._id ?? locals.sessionId,
 			createdAt: new Date(),
 			updatedAt: new Date(),

src/routes/settings/(nav)/assistants/[assistantId]/ReportModal.svelte

@@ -22,10 +22,10 @@
 		}}
 		class="w-full min-w-64 p-4"
 	>
-		<span class="mb-1 text-sm font-semibold">Report an assistant</span>
+		<span class="mb-1 text-sm font-semibold">Report content</span>
 
 		<p class="text-sm text-gray-500">
-			Please provide a brief description of why you are reporting this assistant.
+			Please provide a brief description of why you are reporting this content.
 		</p>
 
 		<textarea

src/routes/settings/+layout.server.ts

@@ -9,9 +9,12 @@ export const load = (async ({ locals, parent }) => {
 	const createdBy = locals.user?._id ?? locals.sessionId;
 	if (createdBy) {
 		const reports = await collections.reports
-			.find<Pick<Report, "assistantId">>({ createdBy }, { projection: { _id: 0, assistantId: 1 } })
+			.find<Pick<Report, "contentId">>(
+				{ createdBy, object: "assistant" },
+				{ projection: { _id: 0, contentId: 1 } }
+			)
 			.toArray();
-		reportsByUser = reports.map((r) => r.assistantId.toString());
+		reportsByUser = reports.map((r) => r.contentId.toString());
 	}
 
 	return {

src/routes/tools/+layout.svelte

@@ -0,0 +1 @@
+<slot />

src/routes/tools/+layout.ts

@@ -0,0 +1,15 @@
+// check if user is earlyAccess else redirect to base
+
+import { base } from "$app/paths";
+import { redirect } from "@sveltejs/kit";
+
+// XXX: feature_flag_tools
+export async function load({ parent }) {
+	const { user } = await parent();
+
+	if (user?.isEarlyAccess) {
+		return {};
+	}
+
+	redirect(302, `${base}/`);
+}

src/routes/tools/+page.server.ts

@@ -0,0 +1,94 @@
+import { authCondition } from "$lib/server/auth.js";
+import { Database, collections } from "$lib/server/database.js";
+import { toolFromConfigs } from "$lib/server/tools/index.js";
+import { SortKey } from "$lib/types/Assistant.js";
+import type { CommunityToolDB } from "$lib/types/Tool.js";
+import type { User } from "$lib/types/User.js";
+import { generateQueryTokens, generateSearchTokens } from "$lib/utils/searchTokens.js";
+import { error } from "@sveltejs/kit";
+import { ObjectId, type Filter } from "mongodb";
+
+const NUM_PER_PAGE = 16;
+
+export const load = async ({ url, locals }) => {
+	// XXX: feature_flag_tools
+	if (!locals.user?.isEarlyAccess) {
+		error(403, "You need to be an early access user to view tools");
+	}
+
+	const username = url.searchParams.get("user");
+	const query = url.searchParams.get("q")?.trim() ?? null;
+
+	const pageIndex = parseInt(url.searchParams.get("p") ?? "0");
+	const sort = url.searchParams.get("sort")?.trim() ?? SortKey.TRENDING;
+	const createdByCurrentUser = locals.user?.username && locals.user.username === username;
+	const activeOnly = url.searchParams.get("active") === "true";
+
+	let user: Pick<User, "_id"> | null = null;
+	if (username) {
+		user = await collections.users.findOne<Pick<User, "_id">>(
+			{ username },
+			{ projection: { _id: 1 } }
+		);
+		if (!user) {
+			error(404, `User "${username}" doesn't exist`);
+		}
+	}
+
+	const settings = await collections.settings.findOne(authCondition(locals));
+
+	if (!settings && activeOnly) {
+		error(404, "No user settings found");
+	}
+
+	const queryTokens = !!query && generateQueryTokens(query);
+
+	const filter: Filter<CommunityToolDB> = {
+		...(!createdByCurrentUser && !activeOnly && { featured: true }),
+		...(user && { createdById: user._id }),
+		...(queryTokens && { searchTokens: { $all: queryTokens } }),
+		...(activeOnly && {
+			_id: {
+				$in: (settings?.tools ?? []).map((key) => {
+					return new ObjectId(key);
+				}),
+			},
+		}),
+	};
+
+	const communityTools = await Database.getInstance()
+		.getCollections()
+		.tools.find(filter)
+		.skip(NUM_PER_PAGE * pageIndex)
+		.sort({
+			...(sort === SortKey.TRENDING && { last24HoursUseCount: -1 }),
+			useCount: -1,
+		})
+		.limit(NUM_PER_PAGE)
+		.toArray();
+
+	const configTools = toolFromConfigs
+		.filter((tool) => !tool?.isHidden)
+		.filter((tool) => {
+			if (queryTokens) {
+				return generateSearchTokens(tool.name).some((token) =>
+					queryTokens.some((queryToken) => queryToken.test(token))
+				);
+			}
+			return true;
+		});
+
+	const tools = [...(pageIndex == 0 && !username ? configTools : []), ...communityTools];
+
+	const numTotalItems =
+		(await Database.getInstance().getCollections().tools.countDocuments(filter)) +
+		toolFromConfigs.length;
+
+	return {
+		tools: JSON.parse(JSON.stringify(tools)) as CommunityToolDB[],
+		numTotalItems,
+		numItemsPerPage: NUM_PER_PAGE,
+		query,
+		sort,
+	};
+};