serve_test.py

import json

from openai import OpenAI

# Initialize the OpenAI client with the local server
client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed",  # API key is not needed for local server
)


def test_chat_completion():
    try:
        print("Sending chat completion request...")
        response = client.chat.completions.create(
            model="unsloth/SmolLM2-135M-Instruct-bnb-4bit",
            messages=[{"role": "user", "content": "Hello"}],
            temperature=0.7,
            max_tokens=50,
        )

        # Print the response
        print("\nResponse:")
        print(response.choices[0].message.content)

        # Print full response object for debugging
        print("\nFull response object:")
        print(json.dumps(response.model_dump(), indent=2))

    except Exception as e:
        print(f"Error occurred: {str(e)}")
        import traceback

        print("\nFull traceback:")
        print(traceback.format_exc())


if __name__ == "__main__":
    print("Testing chat completions endpoint...")
    test_chat_completion()