using qwen2.5

app.py

@@ -1,22 +1,32 @@
 import gradio as gr
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
+from huggingface_hub import hf_hub_download
 
-# ── Model & tokenizer ───────────────────────────────────────────────────────────
-model_name = "microsoft/Phi-3-mini-4k-instruct"  # 3.8 B‑param open instruct model :contentReference[oaicite:0]{index=0}
+# ── Download the GGUF file (only once) ───────────────────────────────────────────
+gguf_path = hf_hub_download(
+    repo_id="Qwen/Qwen2.5-0.5B-Instruct-GGUF",
+    filename="qwen2.5-0.5b-instruct-q5_k_m.gguf"
+)
 
-# seed for reproducibility
-torch.random.manual_seed(0)
+# ── Load tokenizer & model via gguf_file ────────────────────────────────────────
+# Transformers will dequantize the GGUF into fp32 for you
+tokenizer = AutoTokenizer.from_pretrained(
+    "Qwen/Qwen2.5-0.5B-Instruct-GGUF",
+    gguf_file=gguf_path
+)
 
-# load tokenizer (no gating, open access) :contentReference[oaicite:1]{index=1}
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen2.5-0.5B-Instruct-GGUF",
+    gguf_file=gguf_path,
+    device_map="auto",       # spread layers across GPU/CPU
+    torch_dtype=torch.float16 # run in fp16 for speed if GPU supports it
+)
 
-# load as causal‑LM, quantize to int8 to halve memory & speed up inference :contentReference[oaicite:2]{index=2}
-model = AutoModelForCausalLM.from_pretrained(model_name)
-# compile once for ~30% further speed‑up :contentReference[oaicite:3]{index=3}
+# Compile the model for ~20–30% extra speed (PyTorch 2.0+)
 model = torch.compile(model)
 
-# ── Prompt template ──────────────────────────────────────────────────────────────
+# ── Prompt template ─────────────────────────────────────────────────────────────
 prompt_prefix = """
 You are an energy‑saving expert tasked to help households reduce their monthly electricity bills.
 Given the user's appliance usage information (device name, wattage, hours used per day, days used per week):
@@ -27,20 +37,19 @@ Format with bullet points.
 Here is the summary:
 """
 
-# ── Generation function ──────────────────────────────────────────────────────────
+# ── Generation function ─────────────────────────────────────────────────────────
 def generate_recommendation(appliance_info: str) -> str:
     prompt = prompt_prefix + appliance_info + "\n\nRecommendations:"
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     with torch.no_grad():
         outputs = model.generate(
             **inputs,
-            max_new_tokens=150,
+            max_new_tokens=120,
             use_cache=True,
             do_sample=False,
             temperature=0.0
         )
     text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-    # return only the recommendations section
     return text.split("Recommendations:")[-1].strip()
 
 # ── Gradio interface ────────────────────────────────────────────────────────────
@@ -48,8 +57,8 @@ iface = gr.Interface(
     fn=generate_recommendation,
     inputs=gr.Textbox(lines=8, placeholder="e.g. Refrigerator: 150 W, 8 h/day, 7 days/week\n..."),
     outputs="text",
-    title="Energy‑Saving Tips (Phi‑3‑Mini‑4K‑Instruct)",
-    description="Paste your per‑appliance summary to get targeted energy‑saving recommendations."
+    title="Energy‑Saving Tips (Qwen2.5‑0.5B‑Instruct‑GGUF)",
+    description="Provide your appliance usage summary to get targeted, gguf‑powered energy‑saving recommendations."
 )
 
 if __name__ == "__main__":