cpu

inference.py

@@ -29,7 +29,8 @@ def load_model(
 
     model = AutoModelForCausalLM.from_pretrained(
         pretrained_model_name_or_path=model_name,
-        torch_dtype=torch.float32 # Use float32 for CPU
+        torch_dtype=torch.float32, # Use float32 for CPU
+        low_cpu_mem_usage=True  # Helps with memory efficiency
     ).to(device)
 
     model.eval() # Set model to evaluation mode