Update app.py

app.py

@@ -13,9 +13,17 @@ offload_dir = "./offload"
 tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
 # Load the base model with offloading support.
+# base_model = AutoModelForCausalLM.from_pretrained(
+#     model_dir,
+#     device_map="auto",         # Automatically use available devices (GPU/CPU).
+#     offload_folder=offload_dir # Directory to offload layers (for larger models).
+# )
+
 base_model = AutoModelForCausalLM.from_pretrained(
     model_dir,
-    device_map="auto",         # Automatically use available devices (GPU/CPU).
+    device_map="cpu",  # Force CPU usage
+    torch_dtype=torch.float32,  # Use float32 for CPU
+    trust_remote_code=True,
     offload_folder=offload_dir # Directory to offload layers (for larger models).
 )