Update app.py

app.py

@@ -37,7 +37,8 @@ print(f"Loading model: {model_name}")
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
     torch_dtype="auto",
-    # device_map="auto", # device_map="auto" can be helpful for very large models to distribute layers if you have multiple GPUs or for offloading.
+    load_in_8bit=True,
+    device_map="auto", # device_map="auto" can be helpful for very large models to distribute layers if you have multiple GPUs or for offloading.
                          # For single GPU, explicit .to('cuda') is fine.
     # trust_remote_code=True # Removed: Generally not needed for Vicuna/Llama models
 ).to('cuda', torch.bfloat16) # Explicitly using bfloat16 as in original code