Spaces:
Sleeping
Sleeping
File size: 1,506 Bytes
f7c0abb 194ad81 f7c0abb d0fc55f 194ad81 f7c0abb 194ad81 f7c0abb 194ad81 6e02eb7 f7c0abb d0fc55f f7c0abb d0fc55f f7c0abb 194ad81 d0fc55f 194ad81 f7c0abb 194ad81 f7c0abb 194ad81 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
import os
from fastapi import FastAPI, Request
from fastapi.responses import StreamingResponse
from openai import AsyncOpenAI
import asyncio
app = FastAPI()
async def generate_ai_response(prompt: str):
# Get GitHub token from environment variable
token = os.getenv("GITHUB_TOKEN")
endpoint = "https://models.github.ai/inference"
model = "openai/gpt-4-1-mini" # Fixed typo in model name (was gpt-4.1-mini)
# Initialize OpenAI client
client = AsyncOpenAI(base_url=endpoint, api_key=token)
try:
# Create streaming chat completion
stream = await client.chat.completions.create(
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": prompt}
],
temperature=1.0,
top_p=1.0,
model=model,
stream=True
)
# Process the stream
async for chunk in stream:
if chunk.choices and len(chunk.choices) > 0:
content = chunk.choices[0].delta.content or ""
yield content
except Exception as err:
yield f"The sample encountered an error: {err}"
@app.post("/generate")
async def generate_response(request: Request):
data = await request.json()
prompt = data.get("prompt", "what is ai") # Default prompt if none provided
return StreamingResponse(
generate_ai_response(prompt),
media_type="text/event-stream"
)
|