Update app.py

add quantization_config

app.py

@@ -5,6 +5,12 @@ from transformers import AutoProcessor, Llama4ForConditionalGeneration
 
 import torch
 
+from transformers import BitsAndBytesConfig
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    llm_int8_enable_fp32_cpu_offload=True,
+)
+
 #Qwen/Qwen2.5-14B-Instruct-1M
 #Qwen/Qwen2-0.5B
 # model_name = "bartowski/simplescaling_s1-32B-GGUF"
@@ -31,6 +37,7 @@ model = Llama4ForConditionalGeneration.from_pretrained(
     # gguf_file=filename,
     # cache_dir = cache_dir,
     torch_dtype=torch_dtype,
+    quantization_config=bnb_config,
     device_map="auto",
 )