add bnb config

app.py

@@ -5,7 +5,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 import torch
 import os
 
-HF_TOKEN = os.environ.get("HF_TOKEN")  # Make sure to set this in your Space secrets
+HF_TOKEN = os.environ.get("HF_TOKEN") 
 
 title = """
 # Welcome to 🌟Tonic's🫡Command-A
@@ -19,8 +19,8 @@ model_id = "Tonic/c4ai-command-a-03-2025-4bit_fp4"
 quantization_config = BitsAndBytesConfig(
     load_in_4bit=True,  # Enable 4-bit quantization
     bnb_4bit_quant_type="fp4",  # Use FP4 quantization
-    bnb_4bit_use_double_quant=True,  # Optional: double quantization for better precision
-    llm_int8_enable_fp32_cpu_offload=True  # Allow CPU offloading for 32-bit modules
+    bnb_4bit_use_double_quant=True#,  # Optional: double quantization for better precision
+    # llm_int8_enable_fp32_cpu_offload=True  # Allow CPU offloading for 32-bit modules
 )
 
 # Load tokenizer and model
@@ -28,7 +28,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     quantization_config=quantization_config,  # Apply quantization
-    device_map="auto",  # Automatically map to available devices
+    # device_map="auto",  # Automatically map to available devices
     torch_dtype=torch.bfloat16,
     token=HF_TOKEN
 )