Spaces:
Sleeping
Sleeping
File size: 3,222 Bytes
58c3781 3382639 5b1a9aa 58c3781 5b1a9aa 33d6f58 58c3781 a8a9533 58c3781 5b1a9aa 33d6f58 5b1a9aa 58c3781 5b1a9aa 58c3781 4e43408 58c3781 4e43408 58c3781 33d6f58 4e43408 58c3781 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 |
import { z } from "zod";
import type { Endpoint } from "../endpoints";
import { env } from "$env/dynamic/private";
import type { TextGenerationStreamOutput } from "@huggingface/inference";
import { createImageProcessorOptionsValidator } from "../images";
import { endpointMessagesToAnthropicMessages } from "./utils";
import { createDocumentProcessorOptionsValidator } from "../document";
import type { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs";
export const endpointAnthropicParametersSchema = z.object({
weight: z.number().int().positive().default(1),
model: z.any(),
type: z.literal("anthropic"),
baseURL: z.string().url().default("https://api.anthropic.com"),
apiKey: z.string().default(env.ANTHROPIC_API_KEY ?? "sk-"),
defaultHeaders: z.record(z.string()).optional(),
defaultQuery: z.record(z.string()).optional(),
multimodal: z
.object({
image: createImageProcessorOptionsValidator({
supportedMimeTypes: ["image/png", "image/jpeg", "image/webp"],
preferredMimeType: "image/webp",
// The 4 / 3 compensates for the 33% increase in size when converting to base64
maxSizeInMB: (5 / 4) * 3,
maxWidth: 4096,
maxHeight: 4096,
}),
document: createDocumentProcessorOptionsValidator({
supportedMimeTypes: ["application/pdf"],
maxSizeInMB: 32,
}),
})
.default({}),
});
export async function endpointAnthropic(
input: z.input<typeof endpointAnthropicParametersSchema>
): Promise<Endpoint> {
const { baseURL, apiKey, model, defaultHeaders, defaultQuery, multimodal } =
endpointAnthropicParametersSchema.parse(input);
let Anthropic;
try {
Anthropic = (await import("@anthropic-ai/sdk")).default;
} catch (e) {
throw new Error("Failed to import @anthropic-ai/sdk", { cause: e });
}
const anthropic = new Anthropic({
apiKey,
baseURL,
defaultHeaders,
defaultQuery,
});
return async ({ messages, preprompt, generateSettings }) => {
let system = preprompt;
if (messages?.[0]?.from === "system") {
system = messages[0].content;
}
let tokenId = 0;
const parameters = { ...model.parameters, ...generateSettings };
return (async function* () {
const stream = anthropic.messages.stream({
model: model.id ?? model.name,
messages: (await endpointMessagesToAnthropicMessages(
messages,
multimodal
)) as MessageParam[],
max_tokens: parameters?.max_new_tokens,
temperature: parameters?.temperature,
top_p: parameters?.top_p,
top_k: parameters?.top_k,
stop_sequences: parameters?.stop,
system,
});
while (true) {
const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]);
// Stream end
if (result === undefined) {
yield {
token: {
id: tokenId++,
text: "",
logprob: 0,
special: true,
},
generated_text: await stream.finalText(),
details: null,
} satisfies TextGenerationStreamOutput;
return;
}
// Text delta
yield {
token: {
id: tokenId++,
text: result as unknown as string,
special: false,
logprob: 0,
},
generated_text: null,
details: null,
} satisfies TextGenerationStreamOutput;
}
})();
};
}
|