Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -1,12 +1,8 @@
|
|
1 |
-
import requests
|
2 |
import re
|
3 |
import base64
|
4 |
-
import os
|
5 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
|
6 |
from PIL import Image
|
7 |
-
from flask import Flask, render_template, request, redirect, url_for, flash
|
8 |
-
|
9 |
-
app = Flask(__name__)
|
10 |
|
11 |
# Load the Hugging Face model and tokenizer
|
12 |
model_id = "meta-llama/llama-3-2-90b-vision-instruct"
|
@@ -15,16 +11,17 @@ model = AutoModelForCausalLM.from_pretrained(model_id)
|
|
15 |
|
16 |
def input_image_setup(uploaded_file):
|
17 |
"""
|
18 |
-
Encodes the uploaded image file into a base64 string
|
19 |
|
20 |
Parameters:
|
21 |
-
- uploaded_file: File-like object uploaded via
|
22 |
|
23 |
Returns:
|
24 |
-
- encoded_image (str): Base64 encoded string of the image data
|
25 |
"""
|
26 |
if uploaded_file is not None:
|
27 |
-
|
|
|
28 |
encoded_image = base64.b64encode(bytes_data).decode("utf-8")
|
29 |
return encoded_image
|
30 |
else:
|
@@ -42,84 +39,83 @@ def format_response(response_text):
|
|
42 |
response_text = re.sub(r"(\n|\\n)+", r"<br>", response_text)
|
43 |
return response_text
|
44 |
|
45 |
-
def generate_model_response(
|
46 |
-
"""
|
47 |
-
Sends an image and a query to the model and retrieves the description or answer.
|
48 |
-
Formats the response using HTML elements for better presentation.
|
49 |
"""
|
50 |
-
|
51 |
-
input_text = assistant_prompt + "\n\n" + user_query + "\n"
|
52 |
-
|
53 |
-
inputs = tokenizer(input_text, return_tensors="pt")
|
54 |
-
|
55 |
-
try:
|
56 |
-
# Generate the model's response
|
57 |
-
outputs = model.generate(**inputs)
|
58 |
-
raw_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
59 |
-
|
60 |
-
# Format the raw response text using the format_response function
|
61 |
-
formatted_response = format_response(raw_response)
|
62 |
-
return formatted_response
|
63 |
-
except Exception as e:
|
64 |
-
print(f"Error in generating response: {e}")
|
65 |
-
return "<p>An error occurred while generating the response.</p>"
|
66 |
-
|
67 |
-
@app.route("/", methods=["GET", "POST"])
|
68 |
-
def index():
|
69 |
-
if request.method == "POST":
|
70 |
-
user_query = request.form.get("user_query")
|
71 |
-
uploaded_file = request.files.get("file")
|
72 |
-
|
73 |
-
if uploaded_file:
|
74 |
-
encoded_image = input_image_setup(uploaded_file)
|
75 |
-
|
76 |
-
if not encoded_image:
|
77 |
-
flash("Error processing the image. Please try again.", "danger")
|
78 |
-
return redirect(url_for("index"))
|
79 |
-
|
80 |
-
assistant_prompt = """
|
81 |
-
You are an expert nutritionist. Your task is to analyze the food items displayed in the image and provide a detailed nutritional assessment using the following format:
|
82 |
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
Example:
|
88 |
-
* **Salmon**: 6 ounces, 210 calories
|
89 |
-
* **Asparagus**: 3 spears, 25 calories
|
90 |
|
91 |
-
|
|
|
|
|
|
|
|
|
|
|
92 |
|
93 |
-
|
94 |
-
|
|
|
95 |
|
96 |
-
|
|
|
|
|
97 |
|
98 |
-
|
99 |
-
|
|
|
100 |
|
101 |
-
|
102 |
|
103 |
-
|
|
|
104 |
|
105 |
-
|
106 |
-
Actual values may vary depending on factors such as portion size, specific ingredients, preparation methods, and individual variations.
|
107 |
-
For precise dietary advice or medical guidance, consult a qualified nutritionist or healthcare provider.
|
108 |
|
109 |
-
|
110 |
-
|
111 |
|
112 |
-
|
113 |
-
response = generate_model_response(encoded_image, user_query, assistant_prompt)
|
114 |
|
115 |
-
|
116 |
-
return render_template("index.html", user_query=user_query, response=response)
|
117 |
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
|
122 |
-
|
|
|
123 |
|
124 |
-
|
125 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import re
|
2 |
import base64
|
|
|
3 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
4 |
+
import gradio as gr
|
5 |
from PIL import Image
|
|
|
|
|
|
|
6 |
|
7 |
# Load the Hugging Face model and tokenizer
|
8 |
model_id = "meta-llama/llama-3-2-90b-vision-instruct"
|
|
|
11 |
|
12 |
def input_image_setup(uploaded_file):
|
13 |
"""
|
14 |
+
Encodes the uploaded image file into a base64 string.
|
15 |
|
16 |
Parameters:
|
17 |
+
- uploaded_file: File-like object uploaded via Gradio.
|
18 |
|
19 |
Returns:
|
20 |
+
- encoded_image (str): Base64 encoded string of the image data.
|
21 |
"""
|
22 |
if uploaded_file is not None:
|
23 |
+
# Convert the image to bytes and encode in Base64
|
24 |
+
bytes_data = uploaded_file.tobytes()
|
25 |
encoded_image = base64.b64encode(bytes_data).decode("utf-8")
|
26 |
return encoded_image
|
27 |
else:
|
|
|
39 |
response_text = re.sub(r"(\n|\\n)+", r"<br>", response_text)
|
40 |
return response_text
|
41 |
|
42 |
+
def generate_model_response(uploaded_file, user_query):
|
|
|
|
|
|
|
43 |
"""
|
44 |
+
Processes the uploaded image and user query to generate a response from the model.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
45 |
|
46 |
+
Parameters:
|
47 |
+
- uploaded_file: The uploaded image file.
|
48 |
+
- user_query: The user's question about the image.
|
|
|
|
|
|
|
|
|
49 |
|
50 |
+
Returns:
|
51 |
+
- str: The generated response from the model.
|
52 |
+
"""
|
53 |
+
|
54 |
+
# Encode the uploaded image into Base64 format
|
55 |
+
encoded_image = input_image_setup(uploaded_file)
|
56 |
|
57 |
+
# Define the assistant prompt
|
58 |
+
assistant_prompt = """
|
59 |
+
You are an expert nutritionist. Your task is to analyze the food items displayed in the image and provide a detailed nutritional assessment using the following format:
|
60 |
|
61 |
+
1. **Identification**: List each identified food item clearly, one per line.
|
62 |
+
2. **Portion Size & Calorie Estimation**: For each identified food item, specify the portion size and provide an estimated number of calories. Use bullet points with the following structure:
|
63 |
+
- **[Food Item]**: [Portion Size], [Number of Calories] calories
|
64 |
|
65 |
+
Example:
|
66 |
+
* **Salmon**: 6 ounces, 210 calories
|
67 |
+
* **Asparagus**: 3 spears, 25 calories
|
68 |
|
69 |
+
3. **Total Calories**: Provide the total number of calories for all food items.
|
70 |
|
71 |
+
Example:
|
72 |
+
Total Calories: [Number of Calories]
|
73 |
|
74 |
+
4. **Nutrient Breakdown**: Include a breakdown of key nutrients such as **Protein**, **Carbohydrates**, **Fats**, **Vitamins**, and **Minerals**. Use bullet points, and for each nutrient provide details about the contribution of each food item.
|
|
|
|
|
75 |
|
76 |
+
Example:
|
77 |
+
* **Protein**: Salmon (35g), Asparagus (3g), Tomatoes (1g) = [Total Protein]
|
78 |
|
79 |
+
5. **Health Evaluation**: Evaluate the healthiness of the meal in one paragraph.
|
|
|
80 |
|
81 |
+
6. **Disclaimer**: Include the following exact text as a disclaimer:
|
|
|
82 |
|
83 |
+
The nutritional information and calorie estimates provided are approximate and are based on general food data.
|
84 |
+
Actual values may vary depending on factors such as portion size, specific ingredients, preparation methods, and individual variations.
|
85 |
+
For precise dietary advice or medical guidance, consult a qualified nutritionist or healthcare provider.
|
86 |
|
87 |
+
Format your response exactly like the template above to ensure consistency.
|
88 |
+
"""
|
89 |
|
90 |
+
# Prepare input for the model
|
91 |
+
input_text = assistant_prompt + "\n\n" + user_query + "\n"
|
92 |
+
|
93 |
+
# Tokenize input text
|
94 |
+
inputs = tokenizer(input_text, return_tensors="pt")
|
95 |
+
|
96 |
+
try:
|
97 |
+
# Generate response from the model
|
98 |
+
outputs = model.generate(**inputs)
|
99 |
+
|
100 |
+
# Decode and format the model's raw response
|
101 |
+
raw_response = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
102 |
+
formatted_response = format_response(raw_response)
|
103 |
+
|
104 |
+
return formatted_response
|
105 |
+
|
106 |
+
except Exception as e:
|
107 |
+
print(f"Error in generating response: {e}")
|
108 |
+
return "An error occurred while generating the response."
|
109 |
+
|
110 |
+
# Create Gradio interface
|
111 |
+
iface = gr.Interface(
|
112 |
+
fn=generate_model_response,
|
113 |
+
inputs=[
|
114 |
+
gr.Image(type="pil", label="Upload Image"), # Image upload component
|
115 |
+
gr.Textbox(label="User Query", placeholder="Enter your question about the image...")
|
116 |
+
],
|
117 |
+
outputs="html", # Display formatted HTML output
|
118 |
+
)
|
119 |
+
|
120 |
+
# Launch Gradio app
|
121 |
+
iface.launch()
|