Anthropic PDF Beta Support (#1571)

* support anthropic PDF beta

* upstream merge, remove commented out console log line

* Fixing type errors.
the anthropic API does not yet include a "DocumentBlock" for
support PDFs, so an extended type has been added to the endpoint.

* changed document processor to async (matching image processor)

* use the beta api types rather than custom extension

---------

Co-authored-by: Nathan Sarrazin <[email protected]>

package-lock.json

@@ -101,7 +101,7 @@
 				"vitest": "^2.1.4"
 			},
 			"optionalDependencies": {
-				"@anthropic-ai/sdk": "^0.25.0",
+				"@anthropic-ai/sdk": "^0.32.1",
 				"@anthropic-ai/vertex-sdk": "^0.4.1",
 				"@aws-sdk/client-bedrock-runtime": "^3.631.0",
 				"@google-cloud/vertexai": "^1.1.0",
@@ -265,9 +265,9 @@
 			}
 		},
 		"node_modules/@anthropic-ai/sdk": {
-			"version": "0.25.2",
-			"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.25.2.tgz",
-			"integrity": "sha512-F1Hck/asswwidFLtGdMg3XYgRxEUfygNbpkq5KEaEGsHNaSfxeX18/uZGQCL0oQNcj/tYNx8BaFXVwRhFDi45g==",
+			"version": "0.32.1",
+			"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz",
+			"integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==",
 			"optional": true,
 			"dependencies": {
 				"@types/node": "^18.11.18",

package.json

@@ -111,7 +111,7 @@
 		"zod": "^3.22.3"
 	},
 	"optionalDependencies": {
-		"@anthropic-ai/sdk": "^0.25.0",
+		"@anthropic-ai/sdk": "^0.32.1",
 		"@anthropic-ai/vertex-sdk": "^0.4.1",
 		"@aws-sdk/client-bedrock-runtime": "^3.631.0",
 		"@google-cloud/vertexai": "^1.1.0",

src/lib/server/endpoints/anthropic/endpointAnthropic.ts

@@ -4,6 +4,8 @@ import { env } from "$env/dynamic/private";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
 import { createImageProcessorOptionsValidator } from "../images";
 import { endpointMessagesToAnthropicMessages } from "./utils";
+import { createDocumentProcessorOptionsValidator } from "../document";
+import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs";
 
 export const endpointAnthropicParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -23,6 +25,10 @@ export const endpointAnthropicParametersSchema = z.object({
 				maxWidth: 4096,
 				maxHeight: 4096,
 			}),
+			document: createDocumentProcessorOptionsValidator({
+				supportedMimeTypes: ["application/pdf"],
+				maxSizeInMB: 32,
+			}),
 		})
 		.default({}),
 });
@@ -59,7 +65,10 @@ export async function endpointAnthropic(
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
+				messages: (await endpointMessagesToAnthropicMessages(
+					messages,
+					multimodal
+				)) as MessageParam[],
 				max_tokens: parameters?.max_new_tokens,
 				temperature: parameters?.temperature,
 				top_p: parameters?.top_p,

src/lib/server/endpoints/anthropic/endpointAnthropicVertex.ts

@@ -3,6 +3,7 @@ import type { Endpoint } from "../endpoints";
 import type { TextGenerationStreamOutput } from "@huggingface/inference";
 import { createImageProcessorOptionsValidator } from "../images";
 import { endpointMessagesToAnthropicMessages } from "./utils";
+import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs";
 
 export const endpointAnthropicVertexParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -56,7 +57,10 @@ export async function endpointAnthropicVertex(
 		return (async function* () {
 			const stream = anthropic.messages.stream({
 				model: model.id ?? model.name,
-				messages: await endpointMessagesToAnthropicMessages(messages, multimodal),
+				messages: (await endpointMessagesToAnthropicMessages(
+					messages,
+					multimodal
+				)) as MessageParam[],
 				max_tokens: model.parameters?.max_new_tokens,
 				temperature: model.parameters?.temperature,
 				top_p: model.parameters?.top_p,

src/lib/server/endpoints/anthropic/utils.ts

@@ -1,12 +1,17 @@
 import { makeImageProcessor, type ImageProcessorOptions } from "../images";
+import { makeDocumentProcessor, type FileProcessorOptions } from "../document";
 import type { EndpointMessage } from "../endpoints";
 import type { MessageFile } from "$lib/types/Message";
-import type { ImageBlockParam, MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs";
+import type {
+	BetaImageBlockParam,
+	BetaMessageParam,
+	BetaBase64PDFBlock,
+} from "@anthropic-ai/sdk/resources/beta/messages/messages.mjs";
 
 export async function fileToImageBlock(
 	file: MessageFile,
 	opts: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">
-): Promise<ImageBlockParam> {
+): Promise<BetaImageBlockParam> {
 	const processor = makeImageProcessor(opts);
 	const { image, mime } = await processor(file);
 
@@ -20,21 +25,48 @@ export async function fileToImageBlock(
 	};
 }
 
-type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" };
+export async function fileToDocumentBlock(
+	file: MessageFile,
+	opts: FileProcessorOptions<"application/pdf">
+): Promise<BetaBase64PDFBlock> {
+	const processor = makeDocumentProcessor(opts);
+	const { file: document, mime } = await processor(file);
 
+	return {
+		type: "document",
+		source: {
+			type: "base64",
+			media_type: mime,
+			data: document.toString("base64"),
+		},
+	};
+}
+
+type NonSystemMessage = EndpointMessage & { from: "user" | "assistant" };
 export async function endpointMessagesToAnthropicMessages(
 	messages: EndpointMessage[],
-	multimodal: { image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp"> }
-): Promise<MessageParam[]> {
+	multimodal: {
+		image: ImageProcessorOptions<"image/png" | "image/jpeg" | "image/webp">;
+		document?: FileProcessorOptions<"application/pdf">;
+	}
+): Promise<BetaMessageParam[]> {
 	return await Promise.all(
 		messages
 			.filter((message): message is NonSystemMessage => message.from !== "system")
-			.map<Promise<MessageParam>>(async (message) => {
+			.map<Promise<BetaMessageParam>>(async (message) => {
 				return {
 					role: message.from,
 					content: [
 						...(await Promise.all(
-							(message.files ?? []).map((file) => fileToImageBlock(file, multimodal.image))
+							(message.files ?? []).map(async (file) => {
+								if (file.mime.startsWith("image/")) {
+									return fileToImageBlock(file, multimodal.image);
+								} else if (file.mime === "application/pdf" && multimodal.document) {
+									return fileToDocumentBlock(file, multimodal.document);
+								} else {
+									throw new Error(`Unsupported file type: ${file.mime}`);
+								}
+							})
 						)),
 						{ type: "text", text: message.content },
 					],

src/lib/server/endpoints/document.ts

@@ -34,15 +34,21 @@ export type DocumentProcessor<TMimeType extends string = string> = (file: Messag
 	mime: TMimeType;
 };
 
+export type AsyncDocumentProcessor<TMimeType extends string = string> = (
+	file: MessageFile
+) => Promise<{
+	file: Buffer;
+	mime: TMimeType;
+}>;
+
 export function makeDocumentProcessor<TMimeType extends string = string>(
 	options: FileProcessorOptions<TMimeType>
-): DocumentProcessor<TMimeType> {
-	return (file) => {
+): AsyncDocumentProcessor<TMimeType> {
+	return async (file) => {
 		const { supportedMimeTypes, maxSizeInMB } = options;
 		const { mime, value } = file;
 
 		const buffer = Buffer.from(value, "base64");
-
 		const tooLargeInBytes = buffer.byteLength > maxSizeInMB * 1000 * 1000;
 
 		if (tooLargeInBytes) {
@@ -50,7 +56,6 @@ export function makeDocumentProcessor<TMimeType extends string = string>(
 		}
 
 		const outputMime = validateMimeType(supportedMimeTypes, mime);
-
 		return { file: buffer, mime: outputMime };
 	};
 }