add accelerate pip package and add device_map='auto'

app.py

@@ -27,7 +27,7 @@ Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE":
 def model_call(question, document, answer):
     NEW_FORMAT = PROMPT.format(question=question, document=document, answer=answer)
     tokenizer = AutoTokenizer.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct")
-    model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", cache_dir='/tmp/cache', torch_dtype=torch.float16, low_cpu_mem_usage=True)
+    model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", torch_dtype=torch.float16, device_map="auto")
     inputs = tokenizer(NEW_FORMAT, return_tensors="pt")
     model.generate(inputs)
     generated_text = tokenizer.decode(inputs.input_ids[0])

requirements.txt

@@ -1,2 +1,3 @@
 torch==2.4.0
 transformers==4.43.3
+accelerate==0.33.0