Generic Multimodal Support Fixed (#1147)

* feat: multimodal anthropic support

* docs: add claude haiku and multimodal support

* feat: uploaded file detection and image conversion

* fix deps with sharp

* fix resvg deps?

* fix: image conversion, retry with files

* feat: generic image processing and size target

* docs: multimodal review comments

Co-authored-by: Mishig <[email protected]>

* docs: multimodal review comments

Co-authored-by: Mishig <[email protected]>

* feat: review comment resolution

* fix: type error on image params

* feat: add multimodal for vertex ai anthropic

* style: uploadFile timeout number

Co-authored-by: Mishig <[email protected]>

* fix: TGI endpoints expecting multimodal

---------

Co-authored-by: Nathan Sarrazin <[email protected]>
Co-authored-by: Mishig <[email protected]>

README.md

@@ -230,7 +230,7 @@ The following is the default `chatPromptTemplate`, although newlines and indenti
 
 #### Multi modal model
 
-We currently only support IDEFICS as a multimodal model, hosted on TGI. You can enable it by using the following config (if you have a PRO HF Api token):
+We currently support [IDEFICS](https://huggingface.co/blog/idefics) (hosted on TGI), OpenAI and Claude 3 as multimodal models. You can enable it by setting `multimodal: true` in your `MODELS` configuration. For IDEFICS, you must have a [PRO HF Api token](https://huggingface.co/settings/tokens). For OpenAI, see the [OpenAI section](#OpenAI). For Anthropic, see the [Anthropic section](#Anthropic).
 
 ```env
     {
@@ -465,14 +465,34 @@ MODELS=`[
 
 #### Anthropic
 
-We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
+We also support Anthropic models (including multimodal ones via `multmodal: true`) through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
 
 ```
 MODELS=`[
+  {
+      "name": "claude-3-haiku-20240307",
+      "displayName": "Claude 3 Haiku",
+      "description": "Fastest and most compact model for near-instant responsiveness",
+      "multimodal": true,
+      "parameters": {
+        "max_new_tokens": 4096,
+      },
+      "endpoints": [
+        {
+          "type": "anthropic",
+          // optionals
+          "apiKey": "sk-ant-...",
+          "baseURL": "https://api.anthropic.com",
+          "defaultHeaders": {},
+          "defaultQuery": {}
+        }
+      ]
+  },
   {
       "name": "claude-3-sonnet-20240229",
       "displayName": "Claude 3 Sonnet",
       "description": "Ideal balance of intelligence and speed",
+      "multimodal": true,
       "parameters": {
         "max_new_tokens": 4096,
       },
@@ -491,6 +511,7 @@ MODELS=`[
       "name": "claude-3-opus-20240229",
       "displayName": "Claude 3 Opus",
       "description": "Most powerful model for highly complex tasks",
+      "multimodal": true,
       "parameters": {
          "max_new_tokens": 4096
       },
@@ -516,6 +537,7 @@ MODELS=`[
       "name": "claude-3-sonnet@20240229",
       "displayName": "Claude 3 Sonnet",
       "description": "Ideal balance of intelligence and speed",
+      "multimodal": true,
       "parameters": {
         "max_new_tokens": 4096,
       },
@@ -534,6 +556,7 @@ MODELS=`[
       "name": "claude-3-haiku@20240307",
       "displayName": "Claude 3 Haiku",
       "description": "Fastest, most compact model for near-instant responsiveness",
+      "multimodal": true,
       "parameters": {
          "max_new_tokens": 4096
       },

package-lock.json

@@ -13,13 +13,14 @@
 				"@huggingface/inference": "^2.6.3",
 				"@iconify-json/bi": "^1.1.21",
 				"@playwright/browser-chromium": "^1.43.1",
-				"@resvg/resvg-js": "^2.6.0",
+				"@resvg/resvg-js": "^2.6.2",
 				"@xenova/transformers": "^2.16.1",
 				"autoprefixer": "^10.4.14",
 				"browser-image-resizer": "^2.4.1",
 				"date-fns": "^2.29.3",
 				"dotenv": "^16.0.3",
 				"express": "^4.19.2",
+				"file-type": "^19.0.0",
 				"handlebars": "^4.7.8",
 				"highlight.js": "^11.7.0",
 				"image-size": "^1.0.2",
@@ -41,7 +42,7 @@
 				"satori-html": "^0.3.2",
 				"sbd": "^1.0.19",
 				"serpapi": "^1.1.1",
-				"sharp": "^0.33.2",
+				"sharp": "^0.33.3",
 				"tailwind-scrollbar": "^3.0.0",
 				"tailwindcss": "^3.4.0",
 				"uuid": "^9.0.1",
@@ -88,7 +89,7 @@
 				"@google-cloud/vertexai": "^1.1.0",
 				"aws4fetch": "^1.0.17",
 				"cohere-ai": "^7.9.0",
-				"openai": "^4.14.2"
+				"openai": "^4.44.0"
 			}
 		},
 		"node_modules/@alloc/quick-lru": {
@@ -237,9 +238,9 @@
 			}
 		},
 		"node_modules/@emnapi/runtime": {
-			"version": "0.45.0",
-			"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz",
-			"integrity": "sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==",
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz",
+			"integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==",
 			"optional": true,
 			"dependencies": {
 				"tslib": "^2.4.0"
@@ -811,9 +812,9 @@
 			}
 		},
 		"node_modules/@img/sharp-darwin-arm64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz",
-			"integrity": "sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz",
+			"integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==",
 			"cpu": [
 				"arm64"
 			],
@@ -832,13 +833,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-darwin-arm64": "1.0.1"
+				"@img/sharp-libvips-darwin-arm64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-darwin-x64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz",
-			"integrity": "sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz",
+			"integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==",
 			"cpu": [
 				"x64"
 			],
@@ -857,13 +858,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-darwin-x64": "1.0.1"
+				"@img/sharp-libvips-darwin-x64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-libvips-darwin-arm64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz",
-			"integrity": "sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz",
+			"integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==",
 			"cpu": [
 				"arm64"
 			],
@@ -882,9 +883,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-darwin-x64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz",
-			"integrity": "sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz",
+			"integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==",
 			"cpu": [
 				"x64"
 			],
@@ -903,9 +904,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-arm": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz",
-			"integrity": "sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz",
+			"integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==",
 			"cpu": [
 				"arm"
 			],
@@ -924,9 +925,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-arm64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz",
-			"integrity": "sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz",
+			"integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==",
 			"cpu": [
 				"arm64"
 			],
@@ -945,9 +946,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-s390x": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz",
-			"integrity": "sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz",
+			"integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==",
 			"cpu": [
 				"s390x"
 			],
@@ -966,9 +967,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linux-x64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz",
-			"integrity": "sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz",
+			"integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==",
 			"cpu": [
 				"x64"
 			],
@@ -987,9 +988,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linuxmusl-arm64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz",
-			"integrity": "sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz",
+			"integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -1008,9 +1009,9 @@
 			}
 		},
 		"node_modules/@img/sharp-libvips-linuxmusl-x64": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz",
-			"integrity": "sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz",
+			"integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==",
 			"cpu": [
 				"x64"
 			],
@@ -1029,9 +1030,9 @@
 			}
 		},
 		"node_modules/@img/sharp-linux-arm": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz",
-			"integrity": "sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz",
+			"integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==",
 			"cpu": [
 				"arm"
 			],
@@ -1050,13 +1051,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-arm": "1.0.1"
+				"@img/sharp-libvips-linux-arm": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-linux-arm64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz",
-			"integrity": "sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz",
+			"integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==",
 			"cpu": [
 				"arm64"
 			],
@@ -1075,13 +1076,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-arm64": "1.0.1"
+				"@img/sharp-libvips-linux-arm64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-linux-s390x": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz",
-			"integrity": "sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz",
+			"integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==",
 			"cpu": [
 				"s390x"
 			],
@@ -1100,13 +1101,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-s390x": "1.0.1"
+				"@img/sharp-libvips-linux-s390x": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-linux-x64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz",
-			"integrity": "sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz",
+			"integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==",
 			"cpu": [
 				"x64"
 			],
@@ -1125,13 +1126,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linux-x64": "1.0.1"
+				"@img/sharp-libvips-linux-x64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-arm64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz",
-			"integrity": "sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz",
+			"integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==",
 			"cpu": [
 				"arm64"
 			],
@@ -1150,13 +1151,13 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linuxmusl-arm64": "1.0.1"
+				"@img/sharp-libvips-linuxmusl-arm64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-linuxmusl-x64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz",
-			"integrity": "sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz",
+			"integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==",
 			"cpu": [
 				"x64"
 			],
@@ -1175,19 +1176,19 @@
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-libvips-linuxmusl-x64": "1.0.1"
+				"@img/sharp-libvips-linuxmusl-x64": "1.0.2"
 			}
 		},
 		"node_modules/@img/sharp-wasm32": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz",
-			"integrity": "sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz",
+			"integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==",
 			"cpu": [
 				"wasm32"
 			],
 			"optional": true,
 			"dependencies": {
-				"@emnapi/runtime": "^0.45.0"
+				"@emnapi/runtime": "^1.1.0"
 			},
 			"engines": {
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0",
@@ -1200,9 +1201,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-ia32": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz",
-			"integrity": "sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz",
+			"integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==",
 			"cpu": [
 				"ia32"
 			],
@@ -1221,9 +1222,9 @@
 			}
 		},
 		"node_modules/@img/sharp-win32-x64": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz",
-			"integrity": "sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz",
+			"integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==",
 			"cpu": [
 				"x64"
 			],
@@ -1444,31 +1445,31 @@
 			"integrity": "sha512-yvwa+aCyYI/UjeD39BnpMypG8N06l86wIDW1/PAc6ihBRnodIfZDwccxQN3n1t74wduzaz74m4ZMHZnB06567Q=="
 		},
 		"node_modules/@resvg/resvg-js": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.0.tgz",
-			"integrity": "sha512-Tf3YpbBKcQn991KKcw/vg7vZf98v01seSv6CVxZBbRkL/xyjnoYB6KgrFL6zskT1A4dWC/vg77KyNOW+ePaNlA==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js/-/resvg-js-2.6.2.tgz",
+			"integrity": "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q==",
 			"engines": {
 				"node": ">= 10"
 			},
 			"optionalDependencies": {
-				"@resvg/resvg-js-android-arm-eabi": "2.6.0",
-				"@resvg/resvg-js-android-arm64": "2.6.0",
-				"@resvg/resvg-js-darwin-arm64": "2.6.0",
-				"@resvg/resvg-js-darwin-x64": "2.6.0",
-				"@resvg/resvg-js-linux-arm-gnueabihf": "2.6.0",
-				"@resvg/resvg-js-linux-arm64-gnu": "2.6.0",
-				"@resvg/resvg-js-linux-arm64-musl": "2.6.0",
-				"@resvg/resvg-js-linux-x64-gnu": "2.6.0",
-				"@resvg/resvg-js-linux-x64-musl": "2.6.0",
-				"@resvg/resvg-js-win32-arm64-msvc": "2.6.0",
-				"@resvg/resvg-js-win32-ia32-msvc": "2.6.0",
-				"@resvg/resvg-js-win32-x64-msvc": "2.6.0"
+				"@resvg/resvg-js-android-arm-eabi": "2.6.2",
+				"@resvg/resvg-js-android-arm64": "2.6.2",
+				"@resvg/resvg-js-darwin-arm64": "2.6.2",
+				"@resvg/resvg-js-darwin-x64": "2.6.2",
+				"@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2",
+				"@resvg/resvg-js-linux-arm64-gnu": "2.6.2",
+				"@resvg/resvg-js-linux-arm64-musl": "2.6.2",
+				"@resvg/resvg-js-linux-x64-gnu": "2.6.2",
+				"@resvg/resvg-js-linux-x64-musl": "2.6.2",
+				"@resvg/resvg-js-win32-arm64-msvc": "2.6.2",
+				"@resvg/resvg-js-win32-ia32-msvc": "2.6.2",
+				"@resvg/resvg-js-win32-x64-msvc": "2.6.2"
 			}
 		},
 		"node_modules/@resvg/resvg-js-android-arm-eabi": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.0.tgz",
-			"integrity": "sha512-lJnZ/2P5aMocrFMW7HWhVne5gH82I8xH6zsfH75MYr4+/JOaVcGCTEQ06XFohGMdYRP3v05SSPLPvTM/RHjxfA==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.2.tgz",
+			"integrity": "sha512-FrJibrAk6v29eabIPgcTUMPXiEz8ssrAk7TXxsiZzww9UTQ1Z5KAbFJs+Z0Ez+VZTYgnE5IQJqBcoSiMebtPHA==",
 			"cpu": [
 				"arm"
 			],
@@ -1481,9 +1482,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-android-arm64": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.0.tgz",
-			"integrity": "sha512-N527f529bjMwYWShZYfBD60dXA4Fux+D695QsHQ93BDYZSHUoOh1CUGUyICevnTxs7VgEl98XpArmUWBZQVMfQ==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.2.tgz",
+			"integrity": "sha512-VcOKezEhm2VqzXpcIJoITuvUS/fcjIw5NA/w3tjzWyzmvoCdd+QXIqy3FBGulWdClvp4g+IfUemigrkLThSjAQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -1496,9 +1497,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-darwin-arm64": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.0.tgz",
-			"integrity": "sha512-MabUKLVayEwlPo0mIqAmMt+qESN8LltCvv5+GLgVga1avpUrkxj/fkU1TKm8kQegutUjbP/B0QuMuUr0uhF8ew==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-arm64/-/resvg-js-darwin-arm64-2.6.2.tgz",
+			"integrity": "sha512-nmok2LnAd6nLUKI16aEB9ydMC6Lidiiq2m1nEBDR1LaaP7FGs4AJ90qDraxX+CWlVuRlvNjyYJTNv8qFjtL9+A==",
 			"cpu": [
 				"arm64"
 			],
@@ -1511,9 +1512,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-darwin-x64": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.0.tgz",
-			"integrity": "sha512-zrFetdnSw/suXjmyxSjfDV7i61hahv6DDG6kM7BYN2yJ3Es5+BZtqYZTcIWogPJedYKmzN1YTMWGd/3f0ubFiA==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.2.tgz",
+			"integrity": "sha512-GInyZLjgWDfsVT6+SHxQVRwNzV0AuA1uqGsOAW+0th56J7Nh6bHHKXHBWzUrihxMetcFDmQMAX1tZ1fZDYSRsw==",
 			"cpu": [
 				"x64"
 			],
@@ -1526,9 +1527,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm-gnueabihf": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.0.tgz",
-			"integrity": "sha512-sH4gxXt7v7dGwjGyzLwn7SFGvwZG6DQqLaZ11MmzbCwd9Zosy1TnmrMJfn6TJ7RHezmQMgBPi18bl55FZ1AT4A==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.2.tgz",
+			"integrity": "sha512-YIV3u/R9zJbpqTTNwTZM5/ocWetDKGsro0SWp70eGEM9eV2MerWyBRZnQIgzU3YBnSBQ1RcxRZvY/UxwESfZIw==",
 			"cpu": [
 				"arm"
 			],
@@ -1541,9 +1542,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm64-gnu": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.0.tgz",
-			"integrity": "sha512-fCyMncqCJtrlANADIduYF4IfnWQ295UKib7DAxFXQhBsM9PLDTpizr0qemZcCNadcwSVHnAIzL4tliZhCM8P6A==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.2.tgz",
+			"integrity": "sha512-zc2BlJSim7YR4FZDQ8OUoJg5holYzdiYMeobb9pJuGDidGL9KZUv7SbiD4E8oZogtYY42UZEap7dqkkYuA91pg==",
 			"cpu": [
 				"arm64"
 			],
@@ -1556,9 +1557,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-arm64-musl": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.0.tgz",
-			"integrity": "sha512-ouLjTgBQHQyxLht4FdMPTvuY8xzJigM9EM2Tlu0llWkN1mKyTQrvYWi6TA6XnKdzDJHy7ZLpWpjZi7F5+Pg+Vg==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.2.tgz",
+			"integrity": "sha512-3h3dLPWNgSsD4lQBJPb4f+kvdOSJHa5PjTYVsWHxLUzH4IFTJUAnmuWpw4KqyQ3NA5QCyhw4TWgxk3jRkQxEKg==",
 			"cpu": [
 				"arm64"
 			],
@@ -1571,9 +1572,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-x64-gnu": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.0.tgz",
-			"integrity": "sha512-n3zC8DWsvxC1AwxpKFclIPapDFibs5XdIRoV/mcIlxlh0vseW1F49b97F33BtJQRmlntsqqN6GMMqx8byB7B+Q==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.2.tgz",
+			"integrity": "sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==",
 			"cpu": [
 				"x64"
 			],
@@ -1586,9 +1587,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-linux-x64-musl": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.0.tgz",
-			"integrity": "sha512-n4tasK1HOlAxdTEROgYA1aCfsEKk0UOFDNd/AQTTZlTmCbHKXPq+O8npaaKlwXquxlVK8vrkcWbksbiGqbCAcw==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.2.tgz",
+			"integrity": "sha512-UOf83vqTzoYQO9SZ0fPl2ZIFtNIz/Rr/y+7X8XRX1ZnBYsQ/tTb+cj9TE+KHOdmlTFBxhYzVkP2lRByCzqi4jQ==",
 			"cpu": [
 				"x64"
 			],
@@ -1601,9 +1602,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-arm64-msvc": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.0.tgz",
-			"integrity": "sha512-X2+EoBJFwDI5LDVb51Sk7ldnVLitMGr9WwU/i21i3fAeAXZb3hM16k67DeTy16OYkT2dk/RfU1tP1wG+rWbz2Q==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.2.tgz",
+			"integrity": "sha512-7C/RSgCa+7vqZ7qAbItfiaAWhyRSoD4l4BQAbVDqRRsRgY+S+hgS3in0Rxr7IorKUpGE69X48q6/nOAuTJQxeQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -1616,9 +1617,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-ia32-msvc": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.0.tgz",
-			"integrity": "sha512-L7oevWjQoUgK5W1fCKn0euSVemhDXVhrjtwqpc7MwBKKimYeiOshO1Li1pa8bBt5PESahenhWgdB6lav9O0fEg==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.2.tgz",
+			"integrity": "sha512-har4aPAlvjnLcil40AC77YDIk6loMawuJwFINEM7n0pZviwMkMvjb2W5ZirsNOZY4aDbo5tLx0wNMREp5Brk+w==",
 			"cpu": [
 				"ia32"
 			],
@@ -1631,9 +1632,9 @@
 			}
 		},
 		"node_modules/@resvg/resvg-js-win32-x64-msvc": {
-			"version": "2.6.0",
-			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.0.tgz",
-			"integrity": "sha512-8lJlghb+Unki5AyKgsnFbRJwkEj9r1NpwyuBG8yEJiG1W9eEGl03R3I7bsVa3haof/3J1NlWf0rzSa1G++A2iw==",
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.2.tgz",
+			"integrity": "sha512-ZXtYhtUr5SSaBrUDq7DiyjOFJqBVL/dOBN7N/qmi/pO0IgiWW/f/ue3nbvu9joWE5aAKDoIzy/CxsY0suwGosQ==",
 			"cpu": [
 				"x64"
 			],
@@ -2075,6 +2076,11 @@
 				"node": ">=4"
 			}
 		},
+		"node_modules/@tokenizer/token": {
+			"version": "0.3.0",
+			"resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz",
+			"integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A=="
+		},
 		"node_modules/@tootallnate/once": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
@@ -3831,9 +3837,9 @@
 			}
 		},
 		"node_modules/detect-libc": {
-			"version": "2.0.2",
-			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",
-			"integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",
+			"version": "2.0.3",
+			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
+			"integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
 			"engines": {
 				"node": ">=8"
 			}
@@ -4556,6 +4562,22 @@
 				"node": "^10.12.0 || >=12.0.0"
 			}
 		},
+		"node_modules/file-type": {
+			"version": "19.0.0",
+			"resolved": "https://registry.npmjs.org/file-type/-/file-type-19.0.0.tgz",
+			"integrity": "sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==",
+			"dependencies": {
+				"readable-web-to-node-stream": "^3.0.2",
+				"strtok3": "^7.0.0",
+				"token-types": "^5.0.1"
+			},
+			"engines": {
+				"node": ">=18"
+			},
+			"funding": {
+				"url": "https://github.com/sindresorhus/file-type?sponsor=1"
+			}
+		},
 		"node_modules/fill-range": {
 			"version": "7.0.1",
 			"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -6385,16 +6407,15 @@
 			}
 		},
 		"node_modules/openai": {
-			"version": "4.14.2",
-			"resolved": "https://registry.npmjs.org/openai/-/openai-4.14.2.tgz",
-			"integrity": "sha512-JGlm7mMC7J+cyQZnQMOH7daD9cBqqWqLtlBsejElEkgoehPrYfdyxSxIGICz5xk4YimbwI5FlLATSVojLtCKXQ==",
+			"version": "4.47.1",
+			"resolved": "https://registry.npmjs.org/openai/-/openai-4.47.1.tgz",
+			"integrity": "sha512-WWSxhC/69ZhYWxH/OBsLEirIjUcfpQ5+ihkXKp06hmeYXgBBIUCa9IptMzYx6NdkiOCsSGYCnTIsxaic3AjRCQ==",
 			"optional": true,
 			"dependencies": {
 				"@types/node": "^18.11.18",
 				"@types/node-fetch": "^2.6.4",
 				"abort-controller": "^3.0.0",
 				"agentkeepalive": "^4.2.1",
-				"digest-fetch": "^1.3.0",
 				"form-data-encoder": "1.7.2",
 				"formdata-node": "^4.3.2",
 				"node-fetch": "^2.6.7",
@@ -6620,6 +6641,18 @@
 				"node": "*"
 			}
 		},
+		"node_modules/peek-readable": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz",
+			"integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==",
+			"engines": {
+				"node": ">=14.16"
+			},
+			"funding": {
+				"type": "github",
+				"url": "https://github.com/sponsors/Borewit"
+			}
+		},
 		"node_modules/periscopic": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz",
@@ -7475,6 +7508,21 @@
 				"node": ">= 6"
 			}
 		},
+		"node_modules/readable-web-to-node-stream": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz",
+			"integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==",
+			"dependencies": {
+				"readable-stream": "^3.6.0"
+			},
+			"engines": {
+				"node": ">=8"
+			},
+			"funding": {
+				"type": "github",
+				"url": "https://github.com/sponsors/Borewit"
+			}
+		},
 		"node_modules/readdirp": {
 			"version": "3.6.0",
 			"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -7736,12 +7784,9 @@
 			"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
 		},
 		"node_modules/semver": {
-			"version": "7.5.4",
-			"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-			"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
-			"dependencies": {
-				"lru-cache": "^6.0.0"
-			},
+			"version": "7.6.2",
+			"resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
+			"integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
 			"bin": {
 				"semver": "bin/semver.js"
 			},
@@ -7840,42 +7885,42 @@
 			"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
 		},
 		"node_modules/sharp": {
-			"version": "0.33.2",
-			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz",
-			"integrity": "sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==",
+			"version": "0.33.3",
+			"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz",
+			"integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==",
 			"hasInstallScript": true,
 			"dependencies": {
 				"color": "^4.2.3",
-				"detect-libc": "^2.0.2",
-				"semver": "^7.5.4"
+				"detect-libc": "^2.0.3",
+				"semver": "^7.6.0"
 			},
 			"engines": {
-				"libvips": ">=8.15.1",
+				"libvips": ">=8.15.2",
 				"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
 			},
 			"funding": {
 				"url": "https://opencollective.com/libvips"
 			},
 			"optionalDependencies": {
-				"@img/sharp-darwin-arm64": "0.33.2",
-				"@img/sharp-darwin-x64": "0.33.2",
-				"@img/sharp-libvips-darwin-arm64": "1.0.1",
-				"@img/sharp-libvips-darwin-x64": "1.0.1",
-				"@img/sharp-libvips-linux-arm": "1.0.1",
-				"@img/sharp-libvips-linux-arm64": "1.0.1",
-				"@img/sharp-libvips-linux-s390x": "1.0.1",
-				"@img/sharp-libvips-linux-x64": "1.0.1",
-				"@img/sharp-libvips-linuxmusl-arm64": "1.0.1",
-				"@img/sharp-libvips-linuxmusl-x64": "1.0.1",
-				"@img/sharp-linux-arm": "0.33.2",
-				"@img/sharp-linux-arm64": "0.33.2",
-				"@img/sharp-linux-s390x": "0.33.2",
-				"@img/sharp-linux-x64": "0.33.2",
-				"@img/sharp-linuxmusl-arm64": "0.33.2",
-				"@img/sharp-linuxmusl-x64": "0.33.2",
-				"@img/sharp-wasm32": "0.33.2",
-				"@img/sharp-win32-ia32": "0.33.2",
-				"@img/sharp-win32-x64": "0.33.2"
+				"@img/sharp-darwin-arm64": "0.33.3",
+				"@img/sharp-darwin-x64": "0.33.3",
+				"@img/sharp-libvips-darwin-arm64": "1.0.2",
+				"@img/sharp-libvips-darwin-x64": "1.0.2",
+				"@img/sharp-libvips-linux-arm": "1.0.2",
+				"@img/sharp-libvips-linux-arm64": "1.0.2",
+				"@img/sharp-libvips-linux-s390x": "1.0.2",
+				"@img/sharp-libvips-linux-x64": "1.0.2",
+				"@img/sharp-libvips-linuxmusl-arm64": "1.0.2",
+				"@img/sharp-libvips-linuxmusl-x64": "1.0.2",
+				"@img/sharp-linux-arm": "0.33.3",
+				"@img/sharp-linux-arm64": "0.33.3",
+				"@img/sharp-linux-s390x": "0.33.3",
+				"@img/sharp-linux-x64": "0.33.3",
+				"@img/sharp-linuxmusl-arm64": "0.33.3",
+				"@img/sharp-linuxmusl-x64": "0.33.3",
+				"@img/sharp-wasm32": "0.33.3",
+				"@img/sharp-win32-ia32": "0.33.3",
+				"@img/sharp-win32-x64": "0.33.3"
 			}
 		},
 		"node_modules/shebang-command": {
@@ -8187,6 +8232,22 @@
 				"url": "https://github.com/sponsors/antfu"
 			}
 		},
+		"node_modules/strtok3": {
+			"version": "7.0.0",
+			"resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz",
+			"integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==",
+			"dependencies": {
+				"@tokenizer/token": "^0.3.0",
+				"peek-readable": "^5.0.0"
+			},
+			"engines": {
+				"node": ">=14.16"
+			},
+			"funding": {
+				"type": "github",
+				"url": "https://github.com/sponsors/Borewit"
+			}
+		},
 		"node_modules/sucrase": {
 			"version": "3.32.0",
 			"resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz",
@@ -8709,6 +8770,22 @@
 				"node": ">=0.6"
 			}
 		},
+		"node_modules/token-types": {
+			"version": "5.0.1",
+			"resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz",
+			"integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==",
+			"dependencies": {
+				"@tokenizer/token": "^0.3.0",
+				"ieee754": "^1.2.1"
+			},
+			"engines": {
+				"node": ">=14.16"
+			},
+			"funding": {
+				"type": "github",
+				"url": "https://github.com/sponsors/Borewit"
+			}
+		},
 		"node_modules/totalist": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz",

package.json

@@ -57,14 +57,15 @@
 		"@huggingface/hub": "^0.5.1",
 		"@huggingface/inference": "^2.6.3",
 		"@iconify-json/bi": "^1.1.21",
+		"@resvg/resvg-js": "^2.6.2",
 		"@playwright/browser-chromium": "^1.43.1",
-		"@resvg/resvg-js": "^2.6.0",
 		"@xenova/transformers": "^2.16.1",
 		"autoprefixer": "^10.4.14",
 		"browser-image-resizer": "^2.4.1",
 		"date-fns": "^2.29.3",
 		"dotenv": "^16.0.3",
 		"express": "^4.19.2",
+		"file-type": "^19.0.0",
 		"handlebars": "^4.7.8",
 		"highlight.js": "^11.7.0",
 		"image-size": "^1.0.2",
@@ -86,7 +87,7 @@
 		"satori-html": "^0.3.2",
 		"sbd": "^1.0.19",
 		"serpapi": "^1.1.1",
-		"sharp": "^0.33.2",
+		"sharp": "^0.33.3",
 		"tailwind-scrollbar": "^3.0.0",
 		"tailwindcss": "^3.4.0",
 		"uuid": "^9.0.1",
@@ -98,6 +99,6 @@
 		"@google-cloud/vertexai": "^1.1.0",
 		"aws4fetch": "^1.0.17",
 		"cohere-ai": "^7.9.0",
-		"openai": "^4.14.2"
+		"openai": "^4.44.0"
 	}
 }

src/lib/components/chat/ChatMessage.svelte

@@ -308,17 +308,17 @@
 			{#if message.files && message.files.length > 0}
 				<div class="mx-auto grid w-fit grid-cols-2 gap-5 px-5">
 					{#each message.files as file}
-						<!-- handle the case where this is a hash that points to an image in the db, hash is always 64 char long -->
-						{#if file.length === 64}
+						<!-- handle the case where this is a hash that points to an image in the db -->
+						{#if file.type === "hash"}
 							<img
-								src={$page.url.pathname + "/output/" + file}
+								src={$page.url.pathname + "/output/" + file.value}
 								alt="input from user"
 								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
 							/>
 						{:else}
 							<!-- handle the case where this is a base64 encoded image -->
 							<img
-								src={"data:image/*;base64," + file}
+								src={`data:${file.mime};base64,${file.value}`}
 								alt="input from user"
 								class="my-2 aspect-auto max-h-48 rounded-lg shadow-lg"
 							/>

src/lib/components/chat/ChatWindow.svelte

@@ -92,7 +92,9 @@
 		(lastMessage.from === "user" ||
 			lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1);
 
-	$: sources = files.map((file) => file2base64(file));
+	$: sources = files?.map((file) =>
+		file2base64(file).then((value) => ({ type: "base64", value, mime: file.type }))
+	);
 
 	function onShare() {
 		dispatch("share");
@@ -229,13 +231,13 @@
 	<div
 		class="dark:via-gray-80 pointer-events-none absolute inset-x-0 bottom-0 z-0 mx-auto flex w-full max-w-3xl flex-col items-center justify-center bg-gradient-to-t from-white via-white/80 to-white/0 px-3.5 py-4 max-md:border-t max-md:bg-white sm:px-5 md:py-8 xl:max-w-4xl dark:border-gray-800 dark:from-gray-900 dark:to-gray-900/0 max-md:dark:bg-gray-900 [&>*]:pointer-events-auto"
 	>
-		{#if sources.length}
+		{#if sources?.length}
 			<div class="flex flex-row flex-wrap justify-center gap-2.5 max-md:pb-3">
 				{#each sources as source, index}
 					{#await source then src}
 						<div class="relative h-16 w-16 overflow-hidden rounded-lg shadow-lg">
 							<img
-								src={`data:image/*;base64,${src}`}
+								src={`data:${src.mime};base64,${src.value}`}
 								alt="input content"
 								class="h-full w-full rounded-lg bg-gray-400 object-cover dark:bg-gray-900"
 							/>

src/lib/server/endpoints/anthropic/endpointAnthropic.ts

@@ -1,7 +1,9 @@
 import { z } from "zod";
-import { env } from "$env/dynamic/private";
 import type { Endpoint } from "../endpoints";
+import { env } from "$env/dynamic/private";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import { createImageProcessorOptionsValidator } from "../images";
+import { endpointMessagesToAnthropicMessages } from "./utils";
 
 export const endpointAnthropicParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -11,12 +13,24 @@ export const endpointAnthropicParametersSchema = z.object({
 	apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"),
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
+	multimodal: z
+		.object({
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"],
+				preferredMimeType: "image/webp",
+				// The 4 / 3 compensates for the 33% increase in size when converting to base64
+				maxSizeInMB: (5 / 4) * 3,
+				maxWidth: 4096,
+				maxHeight: 4096,
+			}),
+		})
+		.default({}),
 });
 
 export async function endpointAnthropic(
 	input: z.input<typeof endpointAnthropicParametersSchema>
 ): Promise<Endpoint> {
-	const { baseURL, apiKey, model, defaultHeaders, defaultQuery } =
+	const { baseURL, apiKey, model, defaultHeaders, defaultQuery, multimodal } =
 		endpointAnthropicParametersSchema.parse(input);
 	let Anthropic;
 	try {
@@ -38,16 +52,6 @@ export async function endpointAnthropic(
 			system = messages[0].content;
 		}
 
-		const messagesFormatted = messages
-			.filter((message) => message.from !== "system")
-			.map((message) => ({
-				role: message.from,
-				content: message.content,
-			})) as unknown as {
-			role: "user" | "assistant";
-			content: string;
-		}[];
-
 		let tokenId = 0;
 
 		const parameters = { ...model.parameters, ...generateSettings };
@@ -55,7 +59,7 @@ export async function endpointAnthropic(
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: messagesFormatted,
+				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
 				max_tokens: parameters?.max_new_tokens,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,

src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts

@@ -1,6 +1,8 @@
 import { z } from "zod";
 import type { Endpoint } from "../endpoints";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
+import { createImageProcessorOptionsValidator } from "../images";
+import { endpointMessagesToAnthropicMessages } from "./utils";
 
 export const endpointAnthropicVertexParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -10,12 +12,24 @@ export const endpointAnthropicVertexParametersSchema = z.object({
 	projectId: z.string(),
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
+	multimodal: z
+		.object({
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"],
+				preferredMimeType: "image/webp",
+				// The 4 / 3 compensates for the 33% increase in size when converting to base64
+				maxSizeInMB: (5 / 4) * 3,
+				maxWidth: 4096,
+				maxHeight: 4096,
+			}),
+		})
+		.default({}),
 });
 
 export async function endpointAnthropicVertex(
 	input: z.input<typeof endpointAnthropicVertexParametersSchema>
 ): Promise<Endpoint> {
-	const { region, projectId, model, defaultHeaders, defaultQuery } =
+	const { region, projectId, model, defaultHeaders, defaultQuery, multimodal } =
 		endpointAnthropicVertexParametersSchema.parse(input);
 	let AnthropicVertex;
 	try {
@@ -38,21 +52,11 @@ export async function endpointAnthropicVertex(
 			system = messages[0].content;
 		}
 
-		const messagesFormatted = messages
-			.filter((message) => message.from !== "system")
-			.map((message) => ({
-				role: message.from,
-				content: message.content,
-			})) as unknown as {
-			role: "user" | "assistant";
-			content: string;
-		}[];
-
 		let tokenId = 0;
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: messagesFormatted,
+				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
 				max_tokens: model.parameters?.max_new_tokens,
 				temperature: model.parameters?.temperature,
 				top_p: model.parameters?.top_p,

src/lib/server/endpoints/anthropic/utils.ts

@@ -0,0 +1,44 @@
+import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources";
+import { makeImageProcessor, type ImageProcessorOptions } from "../images";
+import type { EndpointMessage } from "../endpoints";
+import type { MessageFile } from "$lib/types/Message";
+
+export async function fileToImageBlock(
+	file: MessageFile,
+	opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">
+): Promise<ImageBlockParam> {
+	const processor = makeImageProcessor(opts);
+	const { image, mime } = await processor(file);
+
+	return {
+		type: "image",
+		source: {
+			type: "base64",
+			media_type: mime,
+			data: image.toString("base64"),
+		},
+	};
+}
+
+type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" };
+
+export async function endpointMessagesToAnthropicMessages(
+	messages: EndpointMessage[],
+	multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> }
+): Promise<MessageParam[]> {
+	return await Promise.all(
+		messages
+			.filter((message): message is NonSystemMessage => message.from !== "system")
+			.map<Promise<MessageParam>>(async (message) => {
+				return {
+					role: message.from,
+					content: [
+						...(await Promise.all(
+							(message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image))
+						)),
+						{ type: "text", text: message.content },
+					],
+				};
+			})
+	);
+}

src/lib/server/endpoints/endpoints.ts

@@ -1,4 +1,5 @@
 import type { Conversation } from "$lib/types/Conversation";
+import type { Message } from "$lib/types/Message";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
 import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi";
 import { z } from "zod";
@@ -25,12 +26,14 @@ import endpointLangserve, {
 	endpointLangserveParametersSchema,
 } from "./langserve/endpointLangserve";
 
+export type EndpointMessage = Omit<Message, "id">;
 // parameters passed when generating text
 export interface EndpointParameters {
-	messages: Omit<Conversation["messages"][0], "id">[];
+	messages: EndpointMessage[];
 	preprompt?: Conversation["preprompt"];
 	continueMessage?: boolean; // used to signal that the last message will be extended
 	generateSettings?: Partial<Model["parameters"]>;
+	isMultimodal?: boolean;
 }
 
 interface CommonEndpoint {

src/lib/server/endpoints/images.ts

@@ -0,0 +1,211 @@
+import type { Sharp } from "sharp";
+import sharp from "sharp";
+import type { MessageFile } from "$lib/types/Message";
+import { z, type util } from "zod";
+
+export interface ImageProcessorOptions<TMimeType extends string = string> {
+	supportedMimeTypes: TMimeType[];
+	preferredMimeType: TMimeType;
+	maxSizeInMB: number;
+	maxWidth: number;
+	maxHeight: number;
+}
+export type ImageProcessor<TMimeType extends string = string> = (file: MessageFile) => Promise<{
+	image: Buffer;
+	mime: TMimeType;
+}>;
+
+export function createImageProcessorOptionsValidator<TMimeType extends string = string>(
+	defaults: ImageProcessorOptions<TMimeType>
+) {
+	return z
+		.object({
+			supportedMimeTypes: z
+				.array(
+					z.enum<string, [TMimeType, ...TMimeType[]]>([
+						defaults.supportedMimeTypes[0],
+						...defaults.supportedMimeTypes.slice(1),
+					])
+				)
+				.default(defaults.supportedMimeTypes),
+			preferredMimeType: z
+				.enum([defaults.supportedMimeTypes[0], ...defaults.supportedMimeTypes.slice(1)])
+				.default(defaults.preferredMimeType as util.noUndefined<TMimeType>),
+			maxSizeInMB: z.number().positive().default(defaults.maxSizeInMB),
+			maxWidth: z.number().int().positive().default(defaults.maxWidth),
+			maxHeight: z.number().int().positive().default(defaults.maxHeight),
+		})
+		.default(defaults);
+}
+
+export function makeImageProcessor<TMimeType extends string = string>(
+	options: ImageProcessorOptions<TMimeType>
+): ImageProcessor<TMimeType> {
+	return async (file) => {
+		const { supportedMimeTypes, preferredMimeType, maxSizeInMB, maxWidth, maxHeight } = options;
+		const { mime, value } = file;
+
+		const buffer = Buffer.from(value, "base64");
+		let sharpInst = sharp(buffer);
+
+		const metadata = await sharpInst.metadata();
+		if (!metadata) throw Error("Failed to read image metadata");
+		const { width, height } = metadata;
+		if (width === undefined || height === undefined) throw Error("Failed to read image size");
+
+		const tooLargeInSize = width > maxWidth || height > maxHeight;
+		const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000;
+
+		const outputMime = chooseMimeType(supportedMimeTypes, preferredMimeType, mime, {
+			preferSizeReduction: tooLargeInBytes,
+		});
+
+		// Resize if necessary
+		if (tooLargeInSize || tooLargeInBytes) {
+			const size = chooseImageSize({
+				mime: outputMime,
+				width,
+				height,
+				maxWidth,
+				maxHeight,
+				maxSizeInMB,
+			});
+			if (size.width !== width || size.height !== height) {
+				sharpInst = resizeImage(sharpInst, size.width, size.height);
+			}
+		}
+
+		// Convert format if necessary
+		// We always want to convert the image when the file was too large in bytes
+		// so we can guarantee that ideal options are used, which are expected when
+		// choosing the image size
+		if (outputMime !== mime || tooLargeInBytes) {
+			sharpInst = convertImage(sharpInst, outputMime);
+		}
+
+		const processedImage = await sharpInst.toBuffer();
+		return { image: processedImage, mime: outputMime };
+	};
+}
+
+const outputFormats = ["png", "jpeg", "webp", "avif", "tiff", "gif"] as const;
+type OutputImgFormat = (typeof outputFormats)[number];
+const isOutputFormat = (format: string): format is (typeof outputFormats)[number] =>
+	outputFormats.includes(format as OutputImgFormat);
+
+export function convertImage(sharpInst: Sharp, outputMime: string): Sharp {
+	const [type, format] = outputMime.split("/");
+	if (type !== "image") throw Error(`Requested non-image mime type: ${outputMime}`);
+	if (!isOutputFormat(format)) {
+		throw Error(`Requested to convert to an unsupported format: ${format}`);
+	}
+
+	return sharpInst[format]();
+}
+
+// heic/heif requires proprietary license
+// TODO: blocking heif may be incorrect considering it also supports av1, so we should instead
+// detect the compression method used via sharp().metadata().compression
+// TODO: consider what to do about animated formats: apng, gif, animated webp, ...
+const blocklistedMimes = ["image/heic", "image/heif"];
+
+/** Sorted from largest to smallest */
+const mimesBySizeDesc = [
+	"image/png",
+	"image/tiff",
+	"image/gif",
+	"image/jpeg",
+	"image/webp",
+	"image/avif",
+];
+
+/**
+ * Defaults to preferred format or uses existing mime if supported
+ * When preferSizeReduction is true, it will choose the smallest format that is supported
+ **/
+function chooseMimeType<T extends readonly string[]>(
+	supportedMimes: T,
+	preferredMime: string,
+	mime: string,
+	{ preferSizeReduction }: { preferSizeReduction: boolean }
+): T[number] {
+	if (!supportedMimes.includes(preferredMime)) {
+		const supportedMimesStr = supportedMimes.join(", ");
+		throw Error(
+			`Preferred format "${preferredMime}" not found in supported mimes: ${supportedMimesStr}`
+		);
+	}
+
+	const [type] = mime.split("/");
+	if (type !== "image") throw Error(`Received non-image mime type: ${mime}`);
+
+	if (supportedMimes.includes(mime) && !preferSizeReduction) return mime;
+
+	if (blocklistedMimes.includes(mime)) throw Error(`Received blocklisted mime type: ${mime}`);
+
+	const smallestMime = mimesBySizeDesc.findLast((m) => supportedMimes.includes(m));
+	return smallestMime ?? preferredMime;
+}
+
+interface ImageSizeOptions {
+	mime: string;
+	width: number;
+	height: number;
+	maxWidth: number;
+	maxHeight: number;
+	maxSizeInMB: number;
+}
+
+/** Resizes the image to fit within the specified size in MB by guessing the output size */
+export function chooseImageSize({
+	mime,
+	width,
+	height,
+	maxWidth,
+	maxHeight,
+	maxSizeInMB,
+}: ImageSizeOptions): { width: number; height: number } {
+	const biggestDiscrepency = Math.max(1, width / maxWidth, height / maxHeight);
+
+	let selectedWidth = Math.ceil(width / biggestDiscrepency);
+	let selectedHeight = Math.ceil(height / biggestDiscrepency);
+
+	do {
+		const estimatedSize = estimateImageSizeInBytes(mime, selectedWidth, selectedHeight);
+		if (estimatedSize < maxSizeInMB * 1024 * 1024) {
+			return { width: selectedWidth, height: selectedHeight };
+		}
+		selectedWidth = Math.floor(selectedWidth / 1.1);
+		selectedHeight = Math.floor(selectedHeight / 1.1);
+	} while (selectedWidth > 1 && selectedHeight > 1);
+
+	throw Error(`Failed to resize image to fit within ${maxSizeInMB}MB`);
+}
+
+const mimeToCompressionRatio: Record<string, number> = {
+	"image/png": 1 / 2,
+	"image/jpeg": 1 / 10,
+	"image/webp": 1 / 4,
+	"image/avif": 1 / 5,
+	"image/tiff": 1,
+	"image/gif": 1 / 5,
+};
+
+/**
+ * Guesses the side of an image in MB based on its format and dimensions
+ * Should guess the worst case
+ **/
+function estimateImageSizeInBytes(mime: string, width: number, height: number): number {
+	const compressionRatio = mimeToCompressionRatio[mime];
+	if (!compressionRatio) throw Error(`Unsupported image format: ${mime}`);
+
+	const bitsPerPixel = 32; // Assuming 32-bit color depth for 8-bit R G B A
+	const bytesPerPixel = bitsPerPixel / 8;
+	const uncompressedSize = width * height * bytesPerPixel;
+
+	return uncompressedSize * compressionRatio;
+}
+
+export function resizeImage(sharpInst: Sharp, maxWidth: number, maxHeight: number): Sharp {
+	return sharpInst.resize({ width: maxWidth, height: maxHeight, fit: "inside" });
+}

src/lib/server/endpoints/openai/endpointOai.ts

@@ -6,6 +6,10 @@ import type { ChatCompletionCreateParamsStreaming } from "openai/resources/chat/
 import { buildPrompt } from "$lib/buildPrompt";
 import { env } from "$env/dynamic/private";
 import type { Endpoint } from "../endpoints";
+import type OpenAI from "openai";
+import { createImageProcessorOptionsValidator, makeImageProcessor } from "../images";
+import type { MessageFile } from "$lib/types/Message";
+import type { EndpointMessage } from "../endpoints";
 
 export const endpointOAIParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -19,13 +23,41 @@ export const endpointOAIParametersSchema = z.object({
 	defaultHeaders: z.record(z.string()).optional(),
 	defaultQuery: z.record(z.string()).optional(),
 	extraBody: z.record(z.any()).optional(),
+	multimodal: z
+		.object({
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: [
+					"image/png",
+					"image/jpeg",
+					"image/webp",
+					"image/avif",
+					"image/tiff",
+					"image/gif",
+				],
+				preferredMimeType: "image/webp",
+				maxSizeInMB: Infinity,
+				maxWidth: 4096,
+				maxHeight: 4096,
+			}),
+		})
+		.default({}),
 });
 
 export async function endpointOai(
 	input: z.input<typeof endpointOAIParametersSchema>
 ): Promise<Endpoint> {
-	const { baseURL, apiKey, completion, model, defaultHeaders, defaultQuery, extraBody } =
-		endpointOAIParametersSchema.parse(input);
+	const {
+		baseURL,
+		apiKey,
+		completion,
+		model,
+		defaultHeaders,
+		defaultQuery,
+		multimodal,
+		extraBody,
+	} = endpointOAIParametersSchema.parse(input);
+
+	/* eslint-disable-next-line no-shadow */
 	let OpenAI;
 	try {
 		OpenAI = (await import("openai")).OpenAI;
@@ -40,6 +72,8 @@ export async function endpointOai(
 		defaultQuery,
 	});
 
+	const imageProcessor = makeImageProcessor(multimodal.image);
+
 	if (completion === "completions") {
 		return async ({ messages, preprompt, continueMessage, generateSettings }) => {
 			const prompt = await buildPrompt({
@@ -69,10 +103,8 @@ export async function endpointOai(
 		};
 	} else if (completion === "chat_completions") {
 		return async ({ messages, preprompt, generateSettings }) => {
-			let messagesOpenAI = messages.map((message) => ({
-				role: message.from,
-				content: message.content,
-			}));
+			let messagesOpenAI: OpenAI.Chat.Completions.ChatCompletionMessageParam[] =
+				await prepareMessages(messages, imageProcessor);
 
 			if (messagesOpenAI?.[0]?.role !== "system") {
 				messagesOpenAI = [{ role: "system", content: "" }, ...messagesOpenAI];
@@ -104,3 +136,39 @@ export async function endpointOai(
 		throw new Error("Invalid completion type");
 	}
 }
+
+async function prepareMessages(
+	messages: EndpointMessage[],
+	imageProcessor: ReturnType<typeof makeImageProcessor>
+): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]> {
+	return Promise.all(
+		messages.map(async (message) => {
+			if (message.from === "user") {
+				return {
+					role: message.from,
+					content: [
+						...(await prepareFiles(imageProcessor, message.files ?? [])),
+						{ type: "text", text: message.content },
+					],
+				};
+			}
+			return {
+				role: message.from,
+				content: message.content,
+			};
+		})
+	);
+}
+
+async function prepareFiles(
+	imageProcessor: ReturnType<typeof makeImageProcessor>,
+	files: MessageFile[]
+): Promise<OpenAI.Chat.Completions.ChatCompletionContentPartImage[]> {
+	const processedFiles = await Promise.all(files.map(imageProcessor));
+	return processedFiles.map((file) => ({
+		type: "image_url" as const,
+		image_url: {
+			url: `data:${file.mime};base64,${file.image.toString("base64")}`,
+		},
+	}));
+}

src/lib/server/endpoints/preprocessMessages.ts

@@ -0,0 +1,56 @@
+import type { Message } from "$lib/types/Message";
+import { format } from "date-fns";
+import type { EndpointMessage } from "./endpoints";
+import { downloadFile } from "../files/downloadFile";
+import type { ObjectId } from "mongodb";
+
+export async function preprocessMessages(
+	messages: Message[],
+	webSearch: Message["webSearch"],
+	convId: ObjectId
+): Promise<EndpointMessage[]> {
+	return Promise.resolve(messages)
+		.then((msgs) => addWebSearchContext(msgs, webSearch))
+		.then((msgs) => downloadFiles(msgs, convId));
+}
+
+function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) {
+	const webSearchContext = webSearch?.contextSources
+		.map(({ context }) => context.trim())
+		.join("\n\n----------\n\n");
+
+	// No web search context available, skip
+	if (!webSearch || !webSearchContext?.trim()) return messages;
+	// No messages available, skip
+	if (messages.length === 0) return messages;
+
+	const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
+	const previousQuestions = messages
+		.filter((el) => el.from === "user")
+		.slice(0, -1)
+		.map((el) => el.content);
+	const currentDate = format(new Date(), "MMMM d, yyyy");
+
+	const finalMessage = {
+		...messages[messages.length - 1],
+		content: `I searched the web using the query: ${webSearch.searchQuery}.
+Today is ${currentDate} and here are the results:
+=====================
+${webSearchContext}
+=====================
+${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
+Answer the question: ${lastQuestion}`,
+	};
+
+	return [...messages.slice(0, -1), finalMessage];
+}
+
+async function downloadFiles(messages: Message[], convId: ObjectId): Promise<EndpointMessage[]> {
+	return Promise.all(
+		messages.map<Promise<EndpointMessage>>((message) =>
+			Promise.all((message.files ?? []).map((file) => downloadFile(file.value, convId))).then(
+				(files) => ({ ...message, files })
+			)
+		)
+	);
+}

src/lib/server/endpoints/tgi/endpointTgi.ts

@@ -1,8 +1,13 @@
 import { env } from "$env/dynamic/private";
 import { buildPrompt } from "$lib/buildPrompt";
 import { textGenerationStream } from "@huggingface/inference";
-import type { Endpoint } from "../endpoints";
+import type { Endpoint, EndpointMessage } from "../endpoints";
 import { z } from "zod";
+import {
+	createImageProcessorOptionsValidator,
+	makeImageProcessor,
+	type ImageProcessor,
+} from "../images";
 
 export const endpointTgiParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -11,14 +16,32 @@ export const endpointTgiParametersSchema = z.object({
 	url: z.string().url(),
 	accessToken: z.string().default(env.HF_TOKEN ?? env.HF_ACCESS_TOKEN),
 	authorization: z.string().optional(),
+	multimodal: z
+		.object({
+			// Assumes IDEFICS
+			image: createImageProcessorOptionsValidator({
+				supportedMimeTypes: ["image/jpeg", "image/webp"],
+				preferredMimeType: "image/webp",
+				maxSizeInMB: 5,
+				maxWidth: 224,
+				maxHeight: 224,
+			}),
+		})
+		.default({}),
 });
 
 export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>): Endpoint {
-	const { url, accessToken, model, authorization } = endpointTgiParametersSchema.parse(input);
+	const { url, accessToken, model, authorization, multimodal } =
+		endpointTgiParametersSchema.parse(input);
+	const imageProcessor = makeImageProcessor(multimodal.image);
+
+	return async ({ messages, preprompt, continueMessage, generateSettings, isMultimodal }) => {
+		const messagesWithResizedFiles = await Promise.all(
+			messages.map((message) => prepareMessage(Boolean(isMultimodal), message, imageProcessor))
+		);
 
-	return async ({ messages, preprompt, continueMessage, generateSettings }) => {
 		const prompt = await buildPrompt({
-			messages,
+			messages: messagesWithResizedFiles,
 			preprompt,
 			model,
 			continueMessage,
@@ -48,4 +71,26 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 	};
 }
 
-export default endpointTgi;
+const whiteImage = {
+	mime: "image/png",
+	image: Buffer.from(
+		"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==",
+		"base64"
+	),
+};
+
+async function prepareMessage(
+	isMultimodal: boolean,
+	message: EndpointMessage,
+	imageProcessor: ImageProcessor
+): Promise<EndpointMessage> {
+	if (!isMultimodal) return message;
+
+	const files = await Promise.all(message.files?.map(imageProcessor) ?? [whiteImage]);
+	const markdowns = files.map(
+		(file) => `![](data:${file.mime};base64,${file.image.toString("base64")})`
+	);
+	const content = message.content + "\n" + markdowns.join("\n ");
+
+	return { ...message, content };
+}

src/lib/server/files/downloadFile.ts

@@ -2,15 +2,16 @@ import { error } from "@sveltejs/kit";
 import { collections } from "$lib/server/database";
 import type { Conversation } from "$lib/types/Conversation";
 import type { SharedConversation } from "$lib/types/SharedConversation";
+import type { MessageFile } from "$lib/types/Message";
 
 export async function downloadFile(
 	sha256: string,
 	convId: Conversation["_id"] | SharedConversation["_id"]
-) {
+): Promise<MessageFile & { type: "base64" }> {
 	const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` });
 	let mime = "";
 
-	const content = await fileId.next().then(async (file) => {
+	const buffer = await fileId.next().then(async (file) => {
 		if (!file) {
 			throw error(404, "File not found");
 		}
@@ -32,5 +33,5 @@ export async function downloadFile(
 		return fileBuffer;
 	});
 
-	return { content, mime };
+	return { type: "base64", value: buffer.toString("base64"), mime };
 }

src/lib/server/files/uploadFile.ts

@@ -1,21 +1,27 @@
 import type { Conversation } from "$lib/types/Conversation";
+import type { MessageFile } from "$lib/types/Message";
 import { sha256 } from "$lib/utils/sha256";
+import { fileTypeFromBuffer } from "file-type";
 import { collections } from "$lib/server/database";
 
-export async function uploadFile(file: Blob, conv: Conversation): Promise<string> {
+export async function uploadFile(file: File, conv: Conversation): Promise<MessageFile> {
 	const sha = await sha256(await file.text());
+	const buffer = await file.arrayBuffer();
+
+	// Attempt to detect the mime type of the file, fallback to the uploaded mime
+	const mime = await fileTypeFromBuffer(buffer).then((fileType) => fileType?.mime ?? file.type);
 
 	const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, {
-		metadata: { conversation: conv._id.toString(), mime: "image/jpeg" },
+		metadata: { conversation: conv._id.toString(), mime },
 	});
 
 	upload.write((await file.arrayBuffer()) as unknown as Buffer);
 	upload.end();
 
-	// only return the filename when upload throws a finish event or a 10s time out occurs
+	// only return the filename when upload throws a finish event or a 20s time out occurs
 	return new Promise((resolve, reject) => {
-		upload.once("finish", () => resolve(sha));
+		upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type }));
 		upload.once("error", reject);
-		setTimeout(() => reject(new Error("Upload timed out")), 10000);
+		setTimeout(() => reject(new Error("Upload timed out")), 20_000);
 	});
 }

src/lib/server/generateFromDefaultEndpoint.ts

@@ -1,12 +1,12 @@
 import { smallModel } from "$lib/server/models";
-import type { Conversation } from "$lib/types/Conversation";
+import type { EndpointMessage } from "./endpoints/endpoints";
 
 export async function generateFromDefaultEndpoint({
 	messages,
 	preprompt,
 	generateSettings,
 }: {
-	messages: Omit<Conversation["messages"][0], "id">[];
+	messages: EndpointMessage[];
 	preprompt?: string;
 	generateSettings?: Record<string, unknown>;
 }): Promise<string> {

src/lib/server/models.ts

@@ -3,7 +3,7 @@ import type { ChatTemplateInput } from "$lib/types/Template";
 import { compileTemplate } from "$lib/utils/template";
 import { z } from "zod";
 import endpoints, { endpointSchema, type Endpoint } from "./endpoints/endpoints";
-import endpointTgi from "./endpoints/tgi/endpointTgi";
+import { endpointTgi } from "./endpoints/tgi/endpointTgi";
 import { sum } from "$lib/utils/sum";
 import { embeddingModels, validateEmbeddingModelByName } from "./embeddingModels";
 

src/lib/server/preprocessMessages.ts

@@ -1,61 +0,0 @@
-import type { Conversation } from "$lib/types/Conversation";
-import type { Message } from "$lib/types/Message";
-import { format } from "date-fns";
-import { downloadFile } from "./files/downloadFile";
-import { logger } from "$lib/server/logger";
-
-export async function preprocessMessages(
-	messages: Message[],
-	webSearch: Message["webSearch"],
-	multimodal: boolean,
-	id: Conversation["_id"]
-): Promise<Message[]> {
-	return await Promise.all(
-		structuredClone(messages).map(async (message, idx) => {
-			const webSearchContext = webSearch?.contextSources
-				.map(({ context }) => context.trim())
-				.join("\n\n----------\n\n");
-
-			// start by adding websearch to the last message
-			if (idx === messages.length - 1 && webSearch && webSearchContext?.trim()) {
-				const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
-				const previousQuestions = messages
-					.filter((el) => el.from === "user")
-					.slice(0, -1)
-					.map((el) => el.content);
-				const currentDate = format(new Date(), "MMMM d, yyyy");
-
-				message.content = `I searched the web using the query: ${webSearch.searchQuery}.
-Today is ${currentDate} and here are the results:
-=====================
-${webSearchContext}
-=====================
-${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
-Answer the question: ${lastQuestion}`;
-			}
-			// handle files if model is multimodal
-			if (multimodal) {
-				if (message.files && message.files.length > 0) {
-					const markdowns = await Promise.all(
-						message.files.map(async (hash) => {
-							try {
-								const { content: image, mime } = await downloadFile(hash, id);
-								const b64 = image.toString("base64");
-								return `![](data:${mime};base64,${b64})})`;
-							} catch (e) {
-								logger.error(e);
-							}
-						})
-					);
-					message.content += markdowns.join("\n ");
-				} else {
-					// if no image, append an empty white image
-					message.content +=
-						"\n![](data:image/png;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAAQABADAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+/igAoAKACgD/2Q==)";
-				}
-			}
-
-			return message;
-		})
-	);
-}

src/lib/server/summarize.ts

@@ -1,6 +1,6 @@
 import { env } from "$env/dynamic/private";
 import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
-import type { Message } from "$lib/types/Message";
+import type { EndpointMessage } from "./endpoints/endpoints";
 import { logger } from "$lib/server/logger";
 
 export async function summarize(prompt: string) {
@@ -8,7 +8,7 @@ export async function summarize(prompt: string) {
 		return prompt.split(/\s+/g).slice(0, 5).join(" ");
 	}
 
-	const messages: Array<Omit<Message, "id">> = [
+	const messages: Array<EndpointMessage> = [
 		{ from: "user", content: "Who is the president of Gabon?" },
 		{ from: "assistant", content: "🇬🇦 President of Gabon" },
 		{ from: "user", content: "Who is Julien Chaumond?" },

src/lib/server/websearch/search/generateQuery.ts

@@ -1,5 +1,6 @@
 import type { Message } from "$lib/types/Message";
 import { format } from "date-fns";
+import type { EndpointMessage } from "../../endpoints/endpoints";
 import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint";
 
 export async function generateQuery(messages: Message[]) {
@@ -9,7 +10,7 @@ export async function generateQuery(messages: Message[]) {
 
 	const lastMessage = userMessages.slice(-1)[0];
 
-	const convQuery: Array<Omit<Message, "id">> = [
+	const convQuery: Array<EndpointMessage> = [
 		{
 			from: "user",
 			content: `Previous Questions:

src/lib/types/Message.ts

@@ -11,7 +11,11 @@ export type Message = Partial<Timestamps> & {
 	webSearchId?: WebSearch["_id"]; // legacy version
 	webSearch?: WebSearch;
 	score?: -1 | 0 | 1;
-	files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading
+	/**
+	 * Either contains the base64 encoded image data
+	 * or the hash of the file stored on the server
+	 **/
+	files?: MessageFile[];
 	interrupted?: boolean;
 
 	// needed for conversation trees
@@ -20,3 +24,9 @@ export type Message = Partial<Timestamps> & {
 	// goes one level deep
 	children?: Message["id"][];
 };
+
+export type MessageFile = {
+	type: "hash" | "base64";
+	value: string;
+	mime: string;
+};

src/lib/utils/messageUpdates.ts

@@ -1,3 +1,4 @@
+import type { MessageFile } from "$lib/types/Message";
 import type { MessageUpdate, TextStreamUpdate } from "$lib/types/MessageUpdate";
 
 type MessageUpdateRequestOptions = {
@@ -7,7 +8,7 @@ type MessageUpdateRequestOptions = {
 	isRetry: boolean;
 	isContinue: boolean;
 	webSearch: boolean;
-	files?: string[];
+	files?: MessageFile[];
 };
 export async function fetchMessageUpdates(
 	conversationId: string,

src/routes/conversation/[id]/+page.svelte

@@ -75,20 +75,10 @@
 			loading = true;
 			pending = true;
 
-			const module = await import("browser-image-resizer");
-			// currently, only IDEFICS is supported by TGI
-			// the size of images is hardcoded to 224x224 in TGI
-			// this will need to be configurable when support for more models is added
-			const resizedImages = await Promise.all(
-				files.map(async (file) => {
-					return await module
-						.readAndCompressImage(file, {
-							maxHeight: 224,
-							maxWidth: 224,
-							quality: 1,
-						})
-						.then(async (el) => await file2base64(el as File));
-				})
+			const base64Files = await Promise.all(
+				(files ?? []).map((file) =>
+					file2base64(file).then((value) => ({ type: "base64" as const, value, mime: file.type }))
+				)
 			);
 
 			let messageToWriteToId: Message["id"] | undefined = undefined;
@@ -120,7 +110,11 @@
 							messages,
 							rootMessageId: data.rootMessageId,
 						},
-						{ from: "user", content: prompt },
+						{
+							from: "user",
+							content: prompt,
+							files: messageToRetry.files,
+						},
 						messageId
 					);
 					messageToWriteToId = addChildren(
@@ -128,7 +122,7 @@
 							messages,
 							rootMessageId: data.rootMessageId,
 						},
-						{ from: "assistant", content: "", files: resizedImages },
+						{ from: "assistant", content: "" },
 						newUserMessageId
 					);
 				} else if (messageToRetry?.from === "assistant") {
@@ -154,7 +148,7 @@
 					{
 						from: "user",
 						content: prompt ?? "",
-						files: resizedImages,
+						files: base64Files,
 						createdAt: new Date(),
 						updatedAt: new Date(),
 					},
@@ -181,6 +175,7 @@
 			}
 
 			messages = [...messages];
+			const userMessage = messages.find((message) => message.id === messageId);
 			const messageToWriteTo = messages.find((message) => message.id === messageToWriteToId);
 			if (!messageToWriteTo) {
 				throw new Error("Message to write to not found");
@@ -198,7 +193,7 @@
 					isRetry,
 					isContinue,
 					webSearch: !hasAssistant && $webSearchParameters.useSearch,
-					files: isRetry ? undefined : resizedImages,
+					files: isRetry ? userMessage?.files : base64Files,
 				},
 				messageUpdatesAbortController.signal
 			).catch((err) => {

src/routes/conversation/[id]/+server.ts

@@ -13,14 +13,13 @@ import { runWebSearch } from "$lib/server/websearch/runWebSearch";
 import { AbortedGenerations } from "$lib/server/abortedGenerations";
 import { summarize } from "$lib/server/summarize";
 import { uploadFile } from "$lib/server/files/uploadFile";
-import sizeof from "image-size";
 import type { Assistant } from "$lib/types/Assistant";
 import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation";
 import { isMessageId } from "$lib/utils/tree/isMessageId";
 import { buildSubtree } from "$lib/utils/tree/buildSubtree.js";
 import { addChildren } from "$lib/utils/tree/addChildren.js";
 import { addSibling } from "$lib/utils/tree/addSibling.js";
-import { preprocessMessages } from "$lib/server/preprocessMessages.js";
+import { preprocessMessages } from "$lib/server/endpoints/preprocessMessages.js";
 import { usageLimits } from "$lib/server/usageLimits";
 import { isURLLocal } from "$lib/server/isURLLocal.js";
 import { logger } from "$lib/server/logger.js";
@@ -134,7 +133,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 		is_retry: isRetry,
 		is_continue: isContinue,
 		web_search: webSearch,
-		files: b64files,
+		files: inputFiles,
 	} = z
 		.object({
 			id: z.string().uuid().refine(isMessageId).optional(), // parent message id to append to for a normal message, or the message id for a retry/continue
@@ -147,44 +146,43 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			is_retry: z.optional(z.boolean()),
 			is_continue: z.optional(z.boolean()),
 			web_search: z.optional(z.boolean()),
-			files: z.optional(z.array(z.string())),
+			files: z.optional(
+				z.array(
+					z.object({
+						type: z.literal("base64").or(z.literal("hash")),
+						value: z.string(),
+						mime: z.string(),
+					})
+				)
+			),
 		})
 		.parse(json);
 
 	if (usageLimits?.messageLength && (newPrompt?.length ?? 0) > usageLimits.messageLength) {
 		throw error(400, "Message too long.");
 	}
-	// files is an array of base64 strings encoding Blob objects
-	// we need to convert this array to an array of File objects
 
-	const files = b64files?.map((file) => {
-		const blob = Buffer.from(file, "base64");
-		return new File([blob], "image.png");
-	});
+	// each file is either:
+	// base64 string requiring upload to the server
+	// hash pointing to an existing file
+	const hashFiles = inputFiles?.filter((file) => file.type === "hash") ?? [];
+	const b64Files =
+		inputFiles
+			?.filter((file) => file.type !== "hash")
+			.map((file) => {
+				const blob = Buffer.from(file.value, "base64");
+				return new File([blob], "file", { type: file.mime });
+			}) ?? [];
 
 	// check sizes
-	if (files) {
-		const filechecks = await Promise.all(
-			files.map(async (file) => {
-				const dimensions = sizeof(Buffer.from(await file.arrayBuffer()));
-				return (
-					file.size > 2 * 1024 * 1024 ||
-					(dimensions.width ?? 0) > 224 ||
-					(dimensions.height ?? 0) > 224
-				);
-			})
-		);
-
-		if (filechecks.some((check) => check)) {
-			throw error(413, "File too large, should be <2MB and 224x224 max.");
-		}
+	// todo: make configurable
+	if (b64Files.some((file) => file.size > 10 * 1024 * 1024)) {
+		throw error(413, "File too large, should be <10MB");
 	}
 
-	let hashes: undefined | string[];
-
-	if (files) {
-		hashes = await Promise.all(files.map(async (file) => await uploadFile(file, conv)));
-	}
+	const uploadedFiles = await Promise.all(b64Files.map((file) => uploadFile(file, conv))).then(
+		(files) => [...files, ...hashFiles]
+	);
 
 	// we will append tokens to the content of this message
 	let messageToWriteToId: Message["id"] | undefined = undefined;
@@ -216,7 +214,13 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			// add a children to that sibling, where we can write to
 			const newUserMessageId = addSibling(
 				conv,
-				{ from: "user", content: newPrompt, createdAt: new Date(), updatedAt: new Date() },
+				{
+					from: "user",
+					content: newPrompt,
+					files: uploadedFiles,
+					createdAt: new Date(),
+					updatedAt: new Date(),
+				},
 				messageId
 			);
 			messageToWriteToId = addChildren(
@@ -224,7 +228,6 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				{
 					from: "assistant",
 					content: "",
-					files: hashes,
 					createdAt: new Date(),
 					updatedAt: new Date(),
 				},
@@ -250,7 +253,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			{
 				from: "user",
 				content: newPrompt ?? "",
-				files: hashes,
+				files: uploadedFiles,
 				createdAt: new Date(),
 				updatedAt: new Date(),
 			},
@@ -411,10 +414,9 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			}
 
 			// inject websearch result & optionally images into the messages
-			const processedMessages = await preprocessMessages(
+			const processedMessages = preprocessMessages(
 				messagesForPrompt,
 				messageToWriteTo.webSearch,
-				model.multimodal,
 				convId
 			);
 
@@ -429,10 +431,11 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			try {
 				const endpoint = await model.getEndpoint();
 				for await (const output of await endpoint({
-					messages: processedMessages,
+					messages: await processedMessages,
 					preprompt,
 					continueMessage: isContinue,
 					generateSettings: assistant?.generateSettings,
+					isMultimodal: model.multimodal,
 				})) {
 					// if not generated_text is here it means the generation is not done
 					if (!output.generated_text) {

src/routes/conversation/[id]/output/[sha256]/+server.ts

@@ -39,9 +39,9 @@ export const GET: RequestHandler = async ({ locals, params }) => {
 		}
 	}
 
-	const { content, mime } = await downloadFile(sha256, params.id);
+	const { value, mime } = await downloadFile(sha256, params.id);
 
-	return new Response(content, {
+	return new Response(Buffer.from(value, "base64"), {
 		headers: {
 			"Content-Type": mime ?? "application/octet-stream",
 		},