Spaces:
Paused
Paused
File size: 2,366 Bytes
3861128 f12f03e 3861128 f12f03e 3861128 f12f03e 3861128 f12f03e 3861128 f12f03e 3861128 f12f03e 3861128 f12f03e 3861128 7741ec9 5aca75a 24fe16c 7741ec9 5aca75a 3861128 5aca75a 3861128 5aca75a 2c756fd 5aca75a 3861128 5aca75a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
import { Message } from '@/types/chat';
import { OpenAIModel } from '@/types/openai';
import { OPENAI_API_HOST } from '../app/const';
import {
ParsedEvent,
ReconnectInterval,
createParser,
} from 'eventsource-parser';
export class LLMError extends Error {
type: string;
param: string;
code: string;
constructor(message: string, type: string, param: string, code: string) {
super(message);
this.name = 'LLMError';
this.type = type;
this.param = param;
this.code = code;
}
}
export const LLMStream = async (
model: OpenAIModel,
systemPrompt: string,
temperature : number,
key: string,
messages: Message[],
) => {
let url = `${OPENAI_API_HOST}/v1/chat/completions`;
const res = await fetch(url, {
headers: {
'Content-Type': 'application/json'
},
method: 'POST',
body: JSON.stringify({
messages: [
{
role: 'system',
content: systemPrompt,
},
...messages,
],
max_tokens: 1000,
temperature: temperature,
stream: true,
}),
});
const encoder = new TextEncoder();
const decoder = new TextDecoder();
if (res.status !== 200) {
const result = await res.json();
if (result.error) {
throw new LLMError(
result.error.message,
result.error.type,
result.error.param,
result.error.code,
);
} else {
throw new Error(
`LLM API returned an error: ${
decoder.decode(result?.value) || result.statusText
}`,
);
}
}
const stream = new ReadableStream({
async start(controller) {
const onParse = (event: ParsedEvent | ReconnectInterval) => {
if (event.type === 'event') {
const data = event.data;
try {
const json = JSON.parse(data);
if (json.choices[0].finish_reason != null) {
controller.close();
return;
}
const text = json.choices[0].delta.content;
const queue = encoder.encode(text);
controller.enqueue(queue);
} catch (e) {
controller.error(e);
}
}
};
const parser = createParser(onParse);
for await (const chunk of res.body as any) {
parser.feed(decoder.decode(chunk));
}
},
});
return stream;
}; |