feat: new tool UI (#1630)

* feat: new tool UI

* wip community tools support

* fix: community tools support

* fix: line heights

* fix: MIME types

* fix: remove debug logs

* fix: make sure file upload is always last

* fix: always enable document parser when a pdf is present in the conversation

* fix: use correct icon for document upload

* fix: make sure community tools use their custom icons correctly

* feat: add button to browse community tools

* tweak buttons

* add some tooltips

* fix: bug in file upload tooltips

* fix: lint

---------

Co-authored-by: Victor Mustar <[email protected]>

chart/env/prod.yaml

@@ -511,7 +511,8 @@ envVars:
         ],
         "outputComponent": "textbox",
         "outputComponentIdx": 0,
-        "showOutput": false
+        "showOutput": false,
+        "isHidden": true
       },
       {
         "_id": "000000000000000000000003",

src/lib/components/HoverTooltip.svelte

@@ -1,11 +1,38 @@
 <script lang="ts">
 	export let label = "";
+	export let position: "top" | "bottom" | "left" | "right" = "bottom";
+	export let TooltipClassNames = "";
+
+	const positionClasses = {
+		top: "bottom-full mb-2",
+		bottom: "top-full mt-2",
+		left: "right-full mr-2 top-1/2 -translate-y-1/2",
+		right: "left-full ml-2 top-1/2 -translate-y-1/2",
+	};
 </script>
 
-<div class="group/tooltip md:relative">
+<div class="group/tooltip inline-block md:relative">
 	<slot />
+
 	<div
-		class="invisible absolute z-10 w-64 whitespace-normal rounded-md bg-black p-2 text-center text-white group-hover/tooltip:visible group-active/tooltip:visible max-sm:left-1/2 max-sm:-translate-x-1/2"
+		class="
+		invisible
+		absolute
+		z-10
+		w-64
+		whitespace-normal
+		rounded-md
+		bg-black
+		p-2
+		text-center
+		text-white
+		group-hover/tooltip:visible
+		group-active/tooltip:visible
+		max-sm:left-1/2
+		max-sm:-translate-x-1/2
+		{positionClasses[position]}
+		{TooltipClassNames}
+	  "
 	>
 		{label}
 	</div>

src/lib/components/ToolLogo.svelte

@@ -13,7 +13,7 @@
 
 	export let color: string;
 	export let icon: string;
-	export let size: "sm" | "md" | "lg" = "md";
+	export let size: "xs" | "sm" | "md" | "lg" = "md";
 
 	$: gradientColor = (() => {
 		switch (color) {
@@ -72,6 +72,8 @@
 
 	$: sizeClass = (() => {
 		switch (size) {
+			case "xs":
+				return "size-4";
 			case "sm":
 				return "size-8";
 			case "md":

src/lib/components/chat/ChatInput.svelte

@@ -2,12 +2,51 @@
 	import { browser } from "$app/environment";
 	import { createEventDispatcher, onMount } from "svelte";
 
+	import HoverTooltip from "$lib/components/HoverTooltip.svelte";
+	import IconInternet from "$lib/components/icons/IconInternet.svelte";
+	import IconImageGen from "$lib/components/icons/IconImageGen.svelte";
+	import IconPaperclip from "$lib/components/icons/IconPaperclip.svelte";
+	import { useSettingsStore } from "$lib/stores/settings";
+	import { webSearchParameters } from "$lib/stores/webSearchParameters";
+	import {
+		documentParserToolId,
+		fetchUrlToolId,
+		imageGenToolId,
+		webSearchToolId,
+	} from "$lib/utils/toolIds";
+	import type { Assistant } from "$lib/types/Assistant";
+	import { page } from "$app/stores";
+	import type { ToolFront } from "$lib/types/Tool";
+	import ToolLogo from "../ToolLogo.svelte";
+	import { goto } from "$app/navigation";
+	import { base } from "$app/paths";
+	import IconAdd from "~icons/carbon/add";
+
+	export let files: File[] = [];
+	export let mimeTypes: string[] = [];
+
 	export let value = "";
-	export let minRows = 1;
-	export let maxRows: null | number = null;
 	export let placeholder = "";
+	export let loading = false;
 	export let disabled = false;
 
+	export let assistant: Assistant | undefined = undefined;
+
+	export let modelHasTools = false;
+	export let modelIsMultimodal = false;
+
+	const onFileChange = async (e: Event) => {
+		if (!e.target) return;
+		const target = e.target as HTMLInputElement;
+		files = [...files, ...(target.files ?? [])];
+
+		if (files.some((file) => file.type.startsWith("application/"))) {
+			await settings.instantSet({
+				tools: [...($settings.tools ?? []), documentParserToolId],
+			});
+		}
+	};
+
 	let textareaElement: HTMLTextAreaElement;
 	let isCompositionOn = false;
 
@@ -28,8 +67,14 @@
 		return /android|webos|iphone|ipad|ipod|blackberry|iemobile|opera mini/i.test(userAgent);
 	}
 
-	$: minHeight = `${1 + minRows * 1.5}em`;
-	$: maxHeight = maxRows ? `${1 + maxRows * 1.5}em` : `auto`;
+	function adjustTextareaHeight() {
+		if (!textareaElement) return;
+		textareaElement.style.height = "auto";
+		const newHeight = Math.min(textareaElement.scrollHeight, parseInt("96em"));
+		textareaElement.style.height = `${newHeight}px`;
+		if (!textareaElement.parentElement) return;
+		textareaElement.parentElement.style.height = `${newHeight}px`;
+	}
 
 	function handleKeydown(event: KeyboardEvent) {
 		if (event.key === "Enter" && !event.shiftKey && !isCompositionOn) {
@@ -48,41 +93,209 @@
 		}
 	}
 
+	const settings = useSettingsStore();
+
+	// tool section
+
+	$: webSearchIsOn = modelHasTools
+		? ($settings.tools?.includes(webSearchToolId) ?? false) ||
+		  ($settings.tools?.includes(fetchUrlToolId) ?? false)
+		: $webSearchParameters.useSearch;
+	$: imageGenIsOn = $settings.tools?.includes(imageGenToolId) ?? false;
+
+	$: documentParserIsOn =
+		modelHasTools && files.length > 0 && files.some((file) => file.type.startsWith("application/"));
+
 	onMount(() => {
 		if (!isVirtualKeyboard()) {
 			textareaElement.focus();
 		}
+		adjustTextareaHeight();
 	});
+
+	$: extraTools = $page.data.tools
+		.filter((t: ToolFront) => $settings.tools?.includes(t._id))
+		.filter(
+			(t: ToolFront) =>
+				![documentParserToolId, imageGenToolId, webSearchToolId, fetchUrlToolId].includes(t._id)
+		) satisfies ToolFront[];
 </script>
 
-<div class="relative min-w-0 flex-1" on:paste>
-	<pre
-		class="scrollbar-custom invisible overflow-x-hidden overflow-y-scroll whitespace-pre-wrap break-words p-3"
-		aria-hidden="true"
-		style="min-height: {minHeight}; max-height: {maxHeight}">{(value || " ") + "\n"}</pre>
-
-	<textarea
-		enterkeyhint={!isVirtualKeyboard() ? "enter" : "send"}
-		tabindex="0"
-		rows="1"
-		class="scrollbar-custom absolute top-0 m-0 h-full w-full resize-none scroll-p-3 overflow-x-hidden overflow-y-scroll border-0 bg-transparent p-3 outline-none focus:ring-0 focus-visible:ring-0 max-sm:p-2.5 max-sm:text-[16px]"
-		class:text-gray-400={disabled}
-		bind:value
-		bind:this={textareaElement}
-		{disabled}
-		on:keydown={handleKeydown}
-		on:compositionstart={() => (isCompositionOn = true)}
-		on:compositionend={() => (isCompositionOn = false)}
-		on:beforeinput
-		{placeholder}
-	/>
+<div class="min-h-full flex-1" on:paste>
+	<div class="relative w-full min-w-0">
+		<textarea
+			enterkeyhint={!isVirtualKeyboard() ? "enter" : "send"}
+			tabindex="0"
+			rows="1"
+			class="scrollbar-custom max-h-[96em] w-full resize-none scroll-p-3 overflow-y-auto overflow-x-hidden border-0 bg-transparent px-3 py-2.5 outline-none focus:ring-0 focus-visible:ring-0 max-sm:p-2.5 max-sm:text-[16px]"
+			class:text-gray-400={disabled}
+			bind:value
+			bind:this={textareaElement}
+			{disabled}
+			on:keydown={handleKeydown}
+			on:compositionstart={() => (isCompositionOn = true)}
+			on:compositionend={() => (isCompositionOn = false)}
+			on:input={adjustTextareaHeight}
+			on:beforeinput
+			{placeholder}
+		/>
+	</div>
+	{#if !assistant}
+		<div
+			class="-ml-0.5 flex flex-wrap items-center justify-start gap-2.5 px-3 pb-2.5 text-gray-500 dark:text-gray-400"
+		>
+			<HoverTooltip
+				label="Search the web"
+				position="top"
+				TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 !mb-0 max-sm:hidden {webSearchIsOn
+					? 'hidden'
+					: ''}"
+			>
+				<button
+					class="base-tool"
+					class:active-tool={webSearchIsOn}
+					disabled={loading}
+					on:click|preventDefault={async () => {
+						if (modelHasTools) {
+							if (webSearchIsOn) {
+								await settings.instantSet({
+									tools: ($settings.tools ?? []).filter(
+										(t) => t !== webSearchToolId && t !== fetchUrlToolId
+									),
+								});
+							} else {
+								await settings.instantSet({
+									tools: [...($settings.tools ?? []), webSearchToolId, fetchUrlToolId],
+								});
+							}
+						} else {
+							$webSearchParameters.useSearch = !webSearchIsOn;
+						}
+					}}
+				>
+					<IconInternet classNames="text-xl" />
+					{#if webSearchIsOn}
+						Search
+					{/if}
+				</button>
+			</HoverTooltip>
+			{#if modelHasTools}
+				<HoverTooltip
+					label="Generate	images"
+					position="top"
+					TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 !mb-0 max-sm:hidden {imageGenIsOn
+						? 'hidden'
+						: ''}"
+				>
+					<button
+						class="base-tool"
+						class:active-tool={imageGenIsOn}
+						disabled={loading}
+						on:click|preventDefault={async () => {
+							if (modelHasTools) {
+								if (imageGenIsOn) {
+									await settings.instantSet({
+										tools: ($settings.tools ?? []).filter((t) => t !== imageGenToolId),
+									});
+								} else {
+									await settings.instantSet({
+										tools: [...($settings.tools ?? []), imageGenToolId],
+									});
+								}
+							}
+						}}
+					>
+						<IconImageGen classNames="text-xl" />
+						{#if imageGenIsOn}
+							Image Gen
+						{/if}
+					</button>
+				</HoverTooltip>
+			{/if}
+			{#if modelHasTools}
+				{#each extraTools as tool}
+					<button
+						class="active-tool base-tool"
+						disabled={loading}
+						on:click|preventDefault={async () => {
+							goto(`${base}/tools/${tool._id}`);
+						}}
+					>
+						<ToolLogo icon={tool.icon} color={tool.color} size="xs" />
+						{tool.displayName}
+					</button>
+				{/each}
+			{/if}
+			{#if modelIsMultimodal || modelHasTools}
+				{@const mimeTypesString = mimeTypes
+					.map((m) => {
+						// if the mime type ends in *, grab the first part so image/* becomes image
+						if (m.endsWith("*")) {
+							return m.split("/")[0];
+						}
+						// otherwise, return the second part for example application/pdf becomes pdf
+						return m.split("/")[1];
+					})
+					.join(", ")}
+				<form class="flex items-center">
+					<HoverTooltip
+						label={`Upload ${mimeTypesString} files`}
+						position="top"
+						TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 !mb-0 max-sm:hidden"
+					>
+						<button
+							class="base-tool relative"
+							class:active-tool={documentParserIsOn}
+							disabled={loading}
+						>
+							<input
+								class="absolute w-full cursor-pointer opacity-0"
+								aria-label="Upload file"
+								type="file"
+								on:change={onFileChange}
+								accept={mimeTypes.join(",")}
+							/>
+							<IconPaperclip classNames="text-xl" />
+							{#if documentParserIsOn}
+								Document Parser
+							{/if}
+						</button>
+					</HoverTooltip>
+				</form>
+			{/if}
+			{#if modelHasTools}
+				<HoverTooltip
+					label="Browse more tools"
+					position="right"
+					TooltipClassNames="text-xs !text-left !w-auto whitespace-nowrap !py-1 max-sm:hidden"
+				>
+					<a
+						class="base-tool flex !size-[20px] items-center justify-center rounded-full bg-white/10"
+						href={`${base}/tools`}
+						title="Browse more tools"
+					>
+						<IconAdd class="text-sm" />
+					</a>
+				</HoverTooltip>
+			{/if}
+		</div>
+	{/if}
+	<slot />
 </div>
 
-<style>
+<style lang="postcss">
 	pre,
 	textarea {
 		font-family: inherit;
 		box-sizing: border-box;
 		line-height: 1.5;
 	}
+
+	.base-tool {
+		@apply flex h-[1.6rem] items-center gap-[.2rem] whitespace-nowrap text-xs outline-none transition-all hover:text-purple-600 focus:outline-none active:outline-none dark:hover:text-gray-300;
+	}
+
+	.active-tool {
+		@apply rounded-full bg-purple-500/15 pl-1 pr-2 text-purple-600 hover:text-purple-600  dark:bg-purple-600/40 dark:text-purple-300;
+	}
 </style>

src/lib/components/chat/ChatWindow.svelte

@@ -13,13 +13,10 @@
 	import ChatInput from "./ChatInput.svelte";
 	import StopGeneratingBtn from "../StopGeneratingBtn.svelte";
 	import type { Model } from "$lib/types/Model";
-	import WebSearchToggle from "../WebSearchToggle.svelte";
-	import ToolsMenu from "../ToolsMenu.svelte";
 	import LoginModal from "../LoginModal.svelte";
 	import { page } from "$app/stores";
 	import FileDropzone from "./FileDropzone.svelte";
 	import RetryBtn from "../RetryBtn.svelte";
-	import UploadBtn from "../UploadBtn.svelte";
 	import file2base64 from "$lib/utils/file2base64";
 	import type { Assistant } from "$lib/types/Assistant";
 	import { base } from "$app/paths";
@@ -35,11 +32,11 @@
 	import { useConvTreeStore } from "$lib/stores/convTree";
 	import UploadedFile from "./UploadedFile.svelte";
 	import { useSettingsStore } from "$lib/stores/settings";
-	import type { ToolFront } from "$lib/types/Tool";
 	import ModelSwitch from "./ModelSwitch.svelte";
 
 	import { fly } from "svelte/transition";
 	import { cubicInOut } from "svelte/easing";
+	import type { ToolFront } from "$lib/types/Tool";
 
 	export let messages: Message[] = [];
 	export let loading = false;
@@ -224,18 +221,25 @@
 
 	const settings = useSettingsStore();
 
-	// active tools are all the checked tools, either from settings or on by default
-	$: activeTools = $page.data.tools.filter((tool: ToolFront) => {
-		if ($page.data?.assistant) {
-			return $page.data.assistant.tools?.includes(tool._id);
-		}
-		return $settings?.tools?.includes(tool._id) ?? tool.isOnByDefault;
-	});
-	$: activeMimeTypes = [
-		...(currentModel.tools ? activeTools.flatMap((tool: ToolFront) => tool.mimeTypes ?? []) : []),
-		...(currentModel.multimodal ? currentModel.multimodalAcceptedMimetypes ?? ["image/*"] : []),
-	];
-
+	$: mimeTypesFromActiveTools = $page.data.tools
+		.filter((tool: ToolFront) => {
+			if ($page.data?.assistant) {
+				return $page.data.assistant.tools?.includes(tool._id);
+			}
+			if (currentModel.tools) {
+				return $settings?.tools?.includes(tool._id) ?? tool.isOnByDefault;
+			}
+			return false;
+		})
+		.flatMap((tool: ToolFront) => tool.mimeTypes ?? []);
+
+	$: activeMimeTypes = Array.from(
+		new Set([
+			...mimeTypesFromActiveTools, // fetch mime types from active tools either from tool settings or active assistant
+			...(currentModel.tools && !$page.data.assistant ? ["application/pdf"] : []), // if its a tool model, we can always enable document parser so we always accept pdfs
+			...(currentModel.multimodal ? currentModel.multimodalAcceptedMimetypes ?? ["image/*"] : []), // if its a multimodal model, we always accept images
+		])
+	);
 	$: isFileUploadEnabled = activeMimeTypes.length > 0;
 </script>
 
@@ -382,13 +386,6 @@
 
 		<div class="w-full">
 			<div class="flex w-full pb-3">
-				{#if !assistant}
-					{#if currentModel.tools}
-						<ToolsMenu {loading} />
-					{:else if $page.data.settings?.searchEnabled}
-						<WebSearchToggle />
-					{/if}
-				{/if}
 				{#if loading}
 					<StopGeneratingBtn classNames="ml-auto" on:click={() => dispatch("stop")} />
 				{:else if lastIsError}
@@ -404,9 +401,6 @@
 					/>
 				{:else}
 					<div class="ml-auto gap-2">
-						{#if isFileUploadEnabled}
-							<UploadBtn bind:files mimeTypes={activeMimeTypes} classNames="ml-auto" />
-						{/if}
 						{#if messages && lastMessage && lastMessage.interrupted && !isReadOnly}
 							<ContinueBtn
 								on:click={() => {
@@ -439,8 +433,12 @@
 							<ChatInput value="Sorry, something went wrong. Please try again." disabled={true} />
 						{:else}
 							<ChatInput
+								{assistant}
 								placeholder={isReadOnly ? "This conversation is read-only." : "Ask anything"}
+								{loading}
 								bind:value={message}
+								bind:files
+								mimeTypes={activeMimeTypes}
 								on:submit={handleSubmit}
 								on:beforeinput={(ev) => {
 									if ($page.data.loginRequired) {
@@ -449,8 +447,9 @@
 									}
 								}}
 								on:paste={onPaste}
-								maxRows={6}
 								disabled={isReadOnly || lastIsError}
+								modelHasTools={currentModel.tools}
+								modelIsMultimodal={currentModel.multimodal}
 							/>
 						{/if}
 

src/lib/components/chat/FileDropzone.svelte

@@ -1,4 +1,6 @@
 <script lang="ts">
+	import { useSettingsStore } from "$lib/stores/settings";
+	import { documentParserToolId } from "$lib/utils/toolIds";
 	import CarbonImage from "~icons/carbon/image";
 	// import EosIconsLoading from "~icons/eos-icons/loading";
 
@@ -8,6 +10,8 @@
 	export let onDrag = false;
 	export let onDragInner = false;
 
+	const settings = useSettingsStore();
+
 	async function dropHandle(event: DragEvent) {
 		event.preventDefault();
 		if (event.dataTransfer && event.dataTransfer.items) {
@@ -32,7 +36,11 @@
 								);
 							})
 						) {
-							setErrorMsg(`Some file type not supported. Only allowed: ${mimeTypes.join(", ")}`);
+							setErrorMsg(
+								`Some file type not supported. Only allowed: ${mimeTypes.join(
+									", "
+								)}. Uploaded document is of type ${file.type}`
+							);
 							files = [];
 							return;
 						}
@@ -46,6 +54,10 @@
 
 						// add the file to the files array
 						files = [...files, file];
+
+						settings.instantSet({
+							tools: [...($settings.tools ?? []), documentParserToolId],
+						});
 					}
 				}
 				onDrag = false;

src/lib/components/icons/IconImageGen.svelte

@@ -0,0 +1,23 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	aria-hidden="true"
+	focusable="false"
+	role="img"
+	width="1em"
+	height="1em"
+	fill="currentColor"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 32 32"
+>
+	<path
+		fill-rule="evenodd"
+		clip-rule="evenodd"
+		d="M6 4H26C26.5304 4 27.0391 4.21071 27.4142 4.58579C27.7893 4.96086 28 5.46957 28 6V26C28 26.5304 27.7893 27.0391 27.4142 27.4142C27.0391 27.7893 26.5304 28 26 28H17C16.4477 28 16 27.5523 16 27C16 26.4477 16.4477 26 17 26H26V24L21 19L19.41 20.59C19.0353 20.9625 18.5284 21.1716 18 21.1716C17.4716 21.1716 16.9647 20.9625 16.59 20.59L14 18L11.1229 15.122C10.7324 14.7315 10.7323 14.0984 11.1227 13.7078C11.5133 13.3169 12.1468 13.3168 12.5375 13.7075L18 19.17L19.59 17.58C19.9647 17.2075 20.4716 16.9984 21 16.9984C21.5284 16.9984 22.0353 17.2075 22.41 17.58L26 21.17V6H6V15C6 15.5523 5.55228 16 5 16C4.44772 16 4 15.5523 4 15V6C4 5.46957 4.21071 4.96086 4.58579 4.58579C4.96086 4.21071 5.46957 4 6 4ZM20.6667 13.4944C20.1734 13.8241 19.5933 14 19 14C18.2044 14 17.4413 13.6839 16.8787 13.1213C16.3161 12.5587 16 11.7957 16 11C16 10.4067 16.1759 9.82664 16.5056 9.33329C16.8352 8.83994 17.3038 8.45543 17.8519 8.22836C18.4001 8.0013 19.0033 7.94189 19.5853 8.05765C20.1672 8.1734 20.7018 8.45912 21.1213 8.87868C21.5409 9.29824 21.8266 9.83279 21.9424 10.4147C22.0581 10.9967 21.9987 11.5999 21.7716 12.1481C21.5446 12.6962 21.1601 13.1648 20.6667 13.4944ZM19.5556 10.1685C19.3911 10.0587 19.1978 10 19 10C18.7348 10 18.4804 10.1054 18.2929 10.2929C18.1054 10.4804 18 10.7348 18 11C18 11.1978 18.0586 11.3911 18.1685 11.5556C18.2784 11.72 18.4346 11.8482 18.6173 11.9239C18.8 11.9996 19.0011 12.0194 19.1951 11.9808C19.3891 11.9422 19.5673 11.847 19.7071 11.7071C19.847 11.5673 19.9422 11.3891 19.9808 11.1951C20.0194 11.0011 19.9996 10.8 19.9239 10.6173C19.8482 10.4346 19.72 10.2784 19.5556 10.1685ZM15.741 24.7381C15.739 24.886 15.6911 25.0295 15.6039 25.149C15.5167 25.2684 15.3946 25.3578 15.2543 25.4048L14.001 25.8181C13.6058 25.9535 13.2491 26.1822 12.961 26.4848C12.6607 26.7773 12.4325 27.1356 12.2943 27.5314L11.861 28.7781C11.8116 28.9156 11.7237 29.036 11.6077 29.1248C11.4847 29.2112 11.338 29.2578 11.1877 29.2581C11.0389 29.261 10.8934 29.2141 10.7743 29.1248C10.6527 29.0372 10.5617 28.9136 10.5143 28.7714L10.0943 27.5181C9.96432 27.1241 9.74415 26.7659 9.45134 26.4719C9.15854 26.1779 8.80117 25.9563 8.40768 25.8248L7.15435 25.4114C7.01338 25.3617 6.89039 25.2712 6.80101 25.1514C6.73433 25.0607 6.6903 24.9553 6.6726 24.8441C6.6549 24.7328 6.66404 24.619 6.69927 24.512C6.73449 24.405 6.79478 24.308 6.8751 24.2291C6.95542 24.1501 7.05345 24.0915 7.16101 24.0581L8.40768 23.6448C8.80559 23.5152 9.16742 23.2938 9.46391 22.9985C9.7604 22.7031 9.98322 22.3422 10.1143 21.9448L10.5277 20.7048C10.5681 20.5663 10.6523 20.4446 10.7677 20.3581C10.8823 20.263 11.0255 20.209 11.1743 20.2048C11.3261 20.1949 11.4766 20.2373 11.601 20.3248C11.7289 20.4059 11.8253 20.5282 11.8743 20.6714L12.2943 21.9381C12.4239 22.336 12.6453 22.6978 12.9407 22.9943C13.236 23.2908 13.5969 23.5136 13.9943 23.6448L15.241 24.0781C15.3813 24.122 15.5032 24.2111 15.5877 24.3314C15.6787 24.4484 15.7322 24.5902 15.741 24.7381ZM9.21774 20.2143C9.21583 20.351 9.17195 20.4838 9.09206 20.5947C9.01216 20.7056 8.90011 20.7892 8.77108 20.8343L7.87774 21.1277C7.62559 21.2166 7.39742 21.3626 7.21108 21.5543C7.02115 21.7421 6.87541 21.9699 6.78441 22.221L6.47108 23.1143C6.42814 23.2389 6.34927 23.3479 6.24441 23.4277C6.13091 23.5072 5.99633 23.5513 5.85774 23.5543C5.72027 23.5519 5.58689 23.5071 5.4759 23.4259C5.36491 23.3448 5.28172 23.2313 5.23774 23.101L4.94441 22.2077C4.85551 21.9555 4.70948 21.7273 4.51774 21.541C4.32993 21.3511 4.1022 21.2053 3.85108 21.1143L2.95108 20.8143C2.82059 20.7723 2.70793 20.6878 2.63108 20.5743C2.54797 20.4652 2.50343 20.3315 2.50441 20.1943C2.50531 20.0554 2.54959 19.9202 2.63108 19.8077C2.71438 19.7006 2.82833 19.6216 2.95774 19.581L3.85108 19.281C4.1022 19.19 4.32993 19.0443 4.51774 18.8543C4.71108 18.6697 4.85708 18.441 4.94441 18.1877L5.24441 17.3077C5.28436 17.1815 5.36095 17.0701 5.46441 16.9877C5.57091 16.9036 5.70206 16.8567 5.83774 16.8543C5.97522 16.8428 6.11231 16.8806 6.22441 16.961C6.33915 17.0368 6.42764 17.1463 6.47774 17.2743L6.77774 18.1877C6.86774 18.4397 7.01374 18.6677 7.20441 18.8543C7.39222 19.0443 7.61995 19.19 7.87108 19.281L8.77108 19.5943C8.89656 19.6412 9.00502 19.7248 9.08232 19.8342C9.15961 19.9436 9.20216 20.0737 9.20441 20.2077L9.21774 20.2143Z"
+		fill="currentColor"
+	/>
+</svg>

src/lib/components/icons/IconInternet.svelte

@@ -12,17 +12,12 @@
 	height="1em"
 	fill="currentColor"
 	preserveAspectRatio="xMidYMid meet"
-	viewBox="0 0 20 20"
+	viewBox="0 0 32 32"
 >
-	><path
+	<path
 		fill-rule="evenodd"
-		d="M1.5 10a8.5 8.5 0 1 0 17 0a8.5 8.5 0 0 0-17 0m16 0a7.5 7.5 0 1 1-15 0a7.5 7.5 0 0 1 15 0"
 		clip-rule="evenodd"
-	/><path
-		fill-rule="evenodd"
-		d="M6.5 10c0 4.396 1.442 8 3.5 8s3.5-3.604 3.5-8s-1.442-8-3.5-8s-3.5 3.604-3.5 8m6 0c0 3.889-1.245 7-2.5 7s-2.5-3.111-2.5-7S8.745 3 10 3s2.5 3.111 2.5 7"
-		clip-rule="evenodd"
-	/><path
-		d="m3.735 5.312l.67-.742c.107.096.221.19.343.281c1.318.988 3.398 1.59 5.665 1.59c1.933 0 3.737-.437 5.055-1.19a5.59 5.59 0 0 0 .857-.597l.65.76c-.298.255-.636.49-1.01.704c-1.477.845-3.452 1.323-5.552 1.323c-2.47 0-4.762-.663-6.265-1.79a5.81 5.81 0 0 1-.413-.34m0 9.389l.67.74c.107-.096.221-.19.343-.28c1.318-.988 3.398-1.59 5.665-1.59c1.933 0 3.737.436 5.055 1.19c.321.184.608.384.857.596l.65-.76a6.583 6.583 0 0 0-1.01-.704c-1.477-.844-3.452-1.322-5.552-1.322c-2.47 0-4.762.663-6.265 1.789c-.146.11-.284.223-.413.34M2 10.5v-1h16v1z"
-	/></svg
->
+		d="M10.5169 17.0606C10.6548 20.6396 11.4841 23.7792 12.7062 25.8143C8.92688 24.5465 6.12113 21.1568 5.70417 17.0606H10.5169ZM12.4184 17.0606C12.5317 19.7958 13.0765 22.1801 13.8416 23.8831C14.7847 25.9823 15.6952 26.3495 15.9998 26.3495C16.3044 26.3495 17.215 25.9823 18.158 23.8831C18.9231 22.1801 19.4679 19.7958 19.5812 17.0606H12.4184ZM19.5894 15.1606H12.4102C12.5026 12.3328 13.0559 9.86568 13.8416 8.11691C14.7847 6.0177 15.6952 5.65049 15.9998 5.65049C16.3044 5.65049 17.215 6.0177 18.158 8.11691C18.9437 9.86568 19.4971 12.3328 19.5894 15.1606ZM21.4827 17.0606C21.3448 20.6397 20.5154 23.7793 19.2934 25.8145C23.0729 24.5468 25.8789 21.157 26.2958 17.0606H21.4827ZM26.316 15.1606H21.4903C21.3787 11.4898 20.5406 8.2625 19.2934 6.1855C23.1407 7.47598 25.9792 10.9654 26.316 15.1606ZM10.5093 15.1606H5.68404C6.02075 10.9655 8.85906 7.47627 12.7062 6.18566C11.459 8.26267 10.6209 11.4899 10.5093 15.1606ZM3.75049 16C3.75049 9.23633 9.23227 3.753 15.9954 3.75049H15.9998H16C22.7652 3.75049 28.2495 9.23478 28.2495 16C28.2495 22.7652 22.7652 28.2495 16 28.2495C9.23478 28.2495 3.75049 22.7652 3.75049 16Z"
+		fill="currentColor"
+	/>
+</svg>

src/lib/components/icons/IconPaperclip.svelte

@@ -0,0 +1,20 @@
+<script lang="ts">
+	export let classNames = "";
+</script>
+
+<svg
+	class={classNames}
+	xmlns="http://www.w3.org/2000/svg"
+	aria-hidden="true"
+	focusable="false"
+	role="img"
+	width="1em"
+	height="1em"
+	fill="currentColor"
+	preserveAspectRatio="xMidYMid meet"
+	viewBox="0 0 32 32"
+	><path
+		d="M19.02 5.57a5.77 5.77 0 1 1 8.56 7.74L16.6 25.45l-.02.01v.01A7.87 7.87 0 0 1 4.92 14.9L12.95 6A1.18 1.18 0 0 1 14.7 7.6l-8.03 8.87a5.51 5.51 0 1 0 8.19 7.4l10.97-12.14a3.41 3.41 0 1 0-5.06-4.58l-9.32 10.3a1.27 1.27 0 1 0 1.88 1.7l6.28-6.94a1.18 1.18 0 0 1 1.75 1.59l-6.28 6.94a3.63 3.63 0 0 1-5.41-4.83l.02-.02 9.33-10.32Z"
+		fill="currentColor"
+	/></svg
+>

src/lib/server/textGeneration/tools.ts

@@ -185,18 +185,21 @@ export async function* runTools(
 		: messages;
 
 	let rawText = "";
+
+	const mappedTools = tools.map((tool) => ({
+		...tool,
+		inputs: tool.inputs.map((input) => ({
+			...input,
+			type: input.type === "file" ? "str" : input.type,
+		})),
+	}));
+
 	// do the function calling bits here
 	for await (const output of await endpoint({
 		messages: formattedMessages,
 		preprompt,
 		generateSettings: { temperature: 0.1, ...assistant?.generateSettings },
-		tools: tools.map((tool) => ({
-			...tool,
-			inputs: tool.inputs.map((input) => ({
-				...input,
-				type: input.type === "file" ? "str" : input.type,
-			})),
-		})),
+		tools: mappedTools,
 		conversationId: conv._id,
 	})) {
 		// model natively supports tool calls

src/lib/types/Tool.ts

@@ -146,7 +146,10 @@ export type CommunityToolEditable = Omit<
 
 export type Tool = ConfigTool | CommunityTool;
 
-export type ToolFront = Pick<Tool, "type" | "name" | "displayName" | "description"> & {
+export type ToolFront = Pick<
+	Tool,
+	"type" | "name" | "displayName" | "description" | "color" | "icon"
+> & {
 	_id: string;
 	isOnByDefault: boolean;
 	isLocked: boolean;

src/lib/utils/toolIds.ts

@@ -0,0 +1,4 @@
+export const webSearchToolId = "00000000000000000000000a";
+export const fetchUrlToolId = "00000000000000000000000b";
+export const imageGenToolId = "000000000000000000000001";
+export const documentParserToolId = "000000000000000000000002";

src/routes/+layout.server.ts

@@ -250,6 +250,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 							toolUseDuration.find(
 								(el) => el.labels.tool === tool._id.toString() && el.labels.quantile === 0.9
 							)?.value ?? 15_000,
+						color: tool.color,
+						icon: tool.icon,
 					} satisfies ToolFront)
 			),
 		communityToolCount: await collections.tools.countDocuments({

src/routes/conversation/[id]/+server.ts

@@ -25,6 +25,7 @@ import { MetricsServer } from "$lib/server/metrics";
 import { textGeneration } from "$lib/server/textGeneration";
 import type { TextGenerationContext } from "$lib/server/textGeneration/types";
 import { logger } from "$lib/server/logger.js";
+import { documentParserToolId } from "$lib/utils/toolIds.js";
 
 export async function POST({ request, locals, params, getClientAddress }) {
 	const id = z.string().parse(params.id);
@@ -190,6 +191,14 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			})
 	);
 
+	// Check for PDF files in the input
+	const hasPdfFiles = inputFiles?.some((file) => file.mime === "application/pdf") ?? false;
+
+	// Check for existing PDF files in the conversation
+	const hasPdfInConversation =
+		conv.messages?.some((msg) => msg.files?.some((file) => file.mime === "application/pdf")) ??
+		false;
+
 	if (usageLimits?.messageLength && (newPrompt?.length ?? 0) > usageLimits.messageLength) {
 		error(400, "Message too long.");
 	}
@@ -442,7 +451,10 @@ export async function POST({ request, locals, params, getClientAddress }) {
 					assistant: undefined,
 					isContinue: isContinue ?? false,
 					webSearch: webSearch ?? false,
-					toolsPreference: toolsPreferences ?? [],
+					toolsPreference: [
+						...(toolsPreferences ?? []),
+						...(hasPdfFiles || hasPdfInConversation ? [documentParserToolId] : []), // Add document parser tool if PDF files are present
+					],
 					promptedAt,
 					ip: getClientAddress(),
 					username: locals.user?.username,