Update app.py

app.py

@@ -34,19 +34,14 @@ model_gpt2 = GPT2LMHeadModel.from_pretrained('gpt2')
 # Create a pipeline for text generation using GPT-2
 text_generator = pipeline("text-generation", model=model_gpt2, tokenizer=tokenizer_gpt2)
 
-# Initialize accelerator with disk offload
-accelerator = Accelerator(cpu=False, disk_offload=True)
-
-# Load the Llama-3 model and tokenizer with disk offload
+# Load the Llama-3 model and tokenizer once during startup
 tokenizer_llama = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B", token=hf_token)
 model_llama = AutoModelForCausalLM.from_pretrained(
     "meta-llama/Meta-Llama-3-8B",
-    torch_dtype='auto',
+    torch_dtype='float16',
     device_map='auto',
-    offload_folder="offload",  # Folder to offload weights to disk
-    offload_state_dict=True,   # Offload state_dict to disk
     token=hf_token
-).to(accelerator.device)
+)
 
 # Define your prompt template
 prompt_template = """..."""  # Your existing prompt template here