load gguf weight

app.py

@@ -7,14 +7,20 @@ import torch
 #Qwen/Qwen2-0.5B
 model_name = "simplescaling/s1.1-32B"
 subfolder = "Qwen-0.5B-GRPO/checkpoint-1868"
+filename = "s1-32B-Q6_K_L.gguf"
+torch_dtype = torch.float32 # could be torch.float16 or torch.bfloat16 too
 
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
     # subfolder=subfolder,
-    torch_dtype=torch.bfloat16,
+    gguf_file=filename,
+    torch_dtype=torch_dtype,
     device_map="auto"
 )
-tokenizer = AutoTokenizer.from_pretrained(model_name, subfolder=subfolder)
+tokenizer = AutoTokenizer.from_pretrained(model_name
+    , gguf_file=filename
+    # , subfolder=subfolder
+    )
 SYSTEM_PROMPT = """
 Respond in the following format:
 <reasoning>