Update app.py

app.py

@@ -1,190 +1,169 @@
-# app.py
 import gradio as gr
 import os
 
-import gradio as gr
-import modelscope_studio as mgr
-from http import HTTPStatus
+os.system('pip install dashscope -U')
+import tempfile
+from pathlib import Path
+import secrets
 import dashscope
-from dashscope import Generation
-from dashscope.api_entities.dashscope_response import Role
-from typing import List, Optional, Tuple, Dict
-from urllib.error import HTTPError
+from dashscope import MultiModalConversation, Generation
+from PIL import Image
 
-default_system = 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.'
 
+# 设置API密钥
 YOUR_API_TOKEN = os.getenv('YOUR_API_TOKEN')
 dashscope.api_key = YOUR_API_TOKEN
-
-History = List[Tuple[str, str]]
-Messages = List[Dict[str, str]]
-
-
-latex_delimiters = [{
-    "left": "\\(",
-    "right": "\\)",
-    "display": True
-}, {
-    "left": "\\begin\{equation\}",
-    "right": "\\end\{equation\}",
-    "display": True
-}, {
-    "left": "\\begin\{align\}",
-    "right": "\\end\{align\}",
-    "display": True
-}, {
-    "left": "\\begin\{alignat\}",
-    "right": "\\end\{alignat\}",
-    "display": True
-}, {
-    "left": "\\begin\{gather\}",
-    "right": "\\end\{gather\}",
-    "display": True
-}, {
-    "left": "\\begin\{CD\}",
-    "right": "\\end\{CD\}",
-    "display": True
-}, {
-    "left": "\\[",
-    "right": "\\]",
-    "display": True
-}]
-
-def clear_session() -> History:
-    return '', []
-
-
-def modify_system_session(system: str) -> str:
-    if system is None or len(system) == 0:
-        system = default_system
-    return system, system, []
-
-
-def history_to_messages(history: History, system: str) -> Messages:
-    messages = [{'role': Role.SYSTEM, 'content': system}]
-    for h in history:
-        messages.append({'role': Role.USER, 'content': h[0].text})
-        messages.append({'role': Role.ASSISTANT, 'content': h[1].text})
-    return messages
-
-
-def messages_to_history(messages: Messages) -> Tuple[str, History]:
-    assert messages[0]['role'] == Role.SYSTEM
-    system = messages[0]['content']
-    history = []
-    for q, r in zip(messages[1::2], messages[2::2]):
-        history.append([q['content'], r['content']])
-    return system, history
-
-
-def model_chat(query: Optional[str], history: Optional[History], system: str, radio: str
-               ) -> Tuple[str, str, History, str]:
-    if query is None:
-        query = ''
-    if history is None:
-        history = []
-    messages = history_to_messages(history, system)
-    messages.append({'role': Role.USER, 'content': query})
+math_messages = []
+def process_image(image, shouldConvert=False):
+    # 获取上传文件的目录
+    global math_messages
+    math_messages = [] # reset when upload image
+    uploaded_file_dir = os.environ.get("GRADIO_TEMP_DIR") or str(
+        Path(tempfile.gettempdir()) / "gradio"
+    )
+    os.makedirs(uploaded_file_dir, exist_ok=True)
+    
+    # 创建临时文件路径
+    name = f"tmp{secrets.token_hex(20)}.jpg"
+    filename = os.path.join(uploaded_file_dir, name)
+    # 保存上传的图片
+    if shouldConvert:
+        new_img = Image.new('RGB', size=(image.width, image.height), color=(255, 255, 255))
+        new_img.paste(image, (0, 0), mask=image)
+        image = new_img
+    image.save(filename)
     
-    label_model = f"qwen2.5-{radio.lower()}-instruct"
+    # 调用qwen-vl-max-0809模型处理图片
+    messages = [{
+        'role': 'system',
+        'content': [{'text': 'You are a helpful assistant.'}]
+    }, {
+        'role': 'user',
+        'content': [
+            {'image': f'file://{filename}'},
+            {'text': 'Please describe the math-related content in this image, ensuring that any LaTeX formulas are correctly transcribed. Non-mathematical details do not need to be described.'}
+        ]
+    }]
     
-    gen = Generation.call(
-        model=label_model,
-        messages=messages,
+    response = MultiModalConversation.call(model='qwen-vl-max-0809', messages=messages)
+    
+    # 清理临时文件
+    os.remove(filename)
+    
+    return response.output.choices[0]["message"]["content"]
+
+def get_math_response(image_description, user_question):
+    global math_messages
+    if not math_messages:
+        math_messages.append({'role': 'system', 'content': 'You are a helpful math assistant.'})
+    math_messages = math_messages[:1]
+    if image_description is not None:
+        content = f'Image description: {image_description}\n\n'
+    else:
+        content = ''
+    query = f"{content}User question: {user_question}"
+    math_messages.append({'role': 'user', 'content': query})
+    response = Generation.call(	
+        model="qwen2-math-72b-instruct",
+        messages=math_messages,	
         result_format='message',
         stream=True
     )
-    for response in gen:
-        if response.status_code == HTTPStatus.OK:
-            role = response.output.choices[0].message.role
-            response = response.output.choices[0].message.content
-            system, history = messages_to_history(messages + [{'role': role, 'content': response}])
-            yield '', history, system
-        else:
-            raise ValueError('Request id: %s, Status code: %s, error code: %s, error message: %s' % (
-                response.request_id, response.status_code,
-                response.code, response.message
-            ))
-
-
-def chiose_radio(radio, system):
-    mark_ = gr.Markdown(value=f"<center><font size=8>Qwen2.5-{radio}-instruct👾</center>")
-    chatbot = mgr.Chatbot(label=f'{radio.lower()}')
-    
-    if system is None or len(system) == 0:
-        system = default_system
-    
-    return mark_, chatbot, system, system, ""
-
-
-def update_other_radios(value, other_radio1, other_radio2):
-    if value == "":
-        if other_radio1 != "":
-            selected = other_radio1
-        else:
-            selected = other_radio2
-        return selected, other_radio1, other_radio2
-    return value, "", ""
-
-
-def main():
-    # 创建两个标签
-    with gr.Blocks() as demo:
-        gr.Markdown("""<center><font size=8>Qwen2.5: A Party of Foundation Models!</center>""")
-        with gr.Row():
-            options_1 = ["72B", "32B", "14B", "7B", "3B", "1.5B", "0.5B"]
-            options_math = ["Math-72B", "Math-7B", "Math-1.5B"]
-            options_coder = ["Coder-7B", "Coder-1.5B"]
-            with gr.Row():
-                radio1 = gr.Radio(choices=options_1, label="Qwen2.5：", value=options_1[0])
-            with gr.Row():
-                radio2 = gr.Radio(choices=options_math, label="Qwen2.5-Math：")
+    answer = None
+    for resp in response:
+        if resp.output is None:
+            continue
+        answer = resp.output.choices[0].message.content
+        yield answer.replace("\\", "\\\\")
+    print(f'query: {query}\nanswer: {answer}')
+    if answer is None:
+        math_messages.pop()
+    else:
+        math_messages.append({'role': 'assistant', 'content': answer})
+
+
+def math_chat_bot(image, sketchpad, question, state):
+    current_tab_index = state["tab_index"]
+    image_description = None
+    # Upload
+    if current_tab_index == 0:
+        if image is not None:
+            image_description = process_image(image)
+    # Sketch
+    elif current_tab_index == 1:
+        print(sketchpad)
+        if sketchpad and sketchpad["composite"]:
+            image_description = process_image(sketchpad["composite"], True)
+    yield from get_math_response(image_description, question)
+
+css = """
+#qwen-md .katex-display { display: inline; }
+#qwen-md .katex-display>.katex { display: inline; }
+#qwen-md .katex-display>.katex>.katex-html { display: inline; }
+"""
+
+def tabs_select(e: gr.SelectData, _state):
+    _state["tab_index"] = e.index
+
+
+# 创建Gradio接口
+with gr.Blocks(css=css) as demo:
+    gr.HTML("""\
+<p align="center"><img src="https://modelscope.oss-cn-beijing.aliyuncs.com/resource/qwen.png" style="height: 60px"/><p>"""
+            """<center><font size=8>📖 Qwen2-Math Demo</center>"""
+            """\
+<center><font size=3>This WebUI is based on Qwen2-VL for OCR and Qwen2-Math for mathematical reasoning. You can input either images or texts of mathematical or arithmetic problems.</center>"""
+            )
+    state = gr.State({"tab_index": 0})
+    with gr.Row():
+        with gr.Column():
+            with gr.Tabs() as input_tabs:
+                with gr.Tab("Upload"):
+                    input_image = gr.Image(type="pil", label="Upload"),
+                with gr.Tab("Sketch"):
+                    input_sketchpad = gr.Sketchpad(type="pil", label="Sketch", layers=False)
+            input_tabs.select(fn=tabs_select, inputs=[state])
+            input_text = gr.Textbox(label="input your question")
             with gr.Row():
-                radio3 = gr.Radio(choices=options_coder, label="Qwen2.5-Coder：")
-        
-        radio = gr.Radio(value=options_1[0], visible=False)
-        radio1.change(fn=update_other_radios, inputs=[radio1, radio2, radio3], outputs=[radio, radio2, radio3])
-        radio2.change(fn=update_other_radios, inputs=[radio2, radio1, radio3], outputs=[radio, radio1, radio3])
-        radio3.change(fn=update_other_radios, inputs=[radio3, radio1, radio2], outputs=[radio, radio1, radio2])
-        
-        with gr.Row():
-            with gr.Accordion():
-                mark_ = gr.Markdown("""<center><font size=8>Qwen2.5-72B-instruct 👾</center>""")
-                with gr.Row():
-                    with gr.Column(scale=3):
-                        system_input = gr.Textbox(value=default_system, lines=1, label='System')
-                    with gr.Column(scale=1):
-                        modify_system = gr.Button("🛠️ Set system prompt and clear history", scale=2)
-                    system_state = gr.Textbox(value=default_system, visible=False)
-                chatbot = mgr.Chatbot(label=options_1[0].lower(), latex_delimiters=latex_delimiters)
-                textbox = gr.Textbox(lines=1, label='Input')
-                
-                with gr.Row():
-                    clear_history = gr.Button("🧹 Clear history")
-                    sumbit = gr.Button("🚀 Send")
-                
-                textbox.submit(model_chat,
-                               inputs=[textbox, chatbot, system_state, radio],
-                               outputs=[textbox, chatbot, system_input])
-                
-                sumbit.click(model_chat,
-                             inputs=[textbox, chatbot, system_state, radio],
-                             outputs=[textbox, chatbot, system_input],
-                             concurrency_limit=5)
-                clear_history.click(fn=clear_session,
-                                    inputs=[],
-                                    outputs=[textbox, chatbot])
-                modify_system.click(fn=modify_system_session,
-                                    inputs=[system_input],
-                                    outputs=[system_state, system_input, chatbot])
-        
-        radio.change(chiose_radio,
-                     inputs=[radio, system_input],
-                     outputs=[mark_, chatbot, system_state, system_input, textbox])
-    
-    demo.queue(api_open=False,default_concurrency_limit=40)
-    demo.launch(max_threads=40)
-
-
-if __name__ == "__main__":
-    main()
+                with gr.Column():
+                    clear_btn = gr.ClearButton(
+                        [*input_image, input_sketchpad, input_text])
+                with gr.Column():
+                    submit_btn = gr.Button("Submit", variant="primary")
+        with gr.Column():
+            output_md = gr.Markdown(label="answer",
+                                    latex_delimiters=[{
+                                        "left": "\\(",
+                                        "right": "\\)",
+                                        "display": True
+                                    }, {
+                                        "left": "\\begin\{equation\}",
+                                        "right": "\\end\{equation\}",
+                                        "display": True
+                                    }, {
+                                        "left": "\\begin\{align\}",
+                                        "right": "\\end\{align\}",
+                                        "display": True
+                                    }, {
+                                        "left": "\\begin\{alignat\}",
+                                        "right": "\\end\{alignat\}",
+                                        "display": True
+                                    }, {
+                                        "left": "\\begin\{gather\}",
+                                        "right": "\\end\{gather\}",
+                                        "display": True
+                                    }, {
+                                        "left": "\\begin\{CD\}",
+                                        "right": "\\end\{CD\}",
+                                        "display": True
+                                    }, {
+                                        "left": "\\[",
+                                        "right": "\\]",
+                                        "display": True
+                                    }],
+                                    elem_id="qwen-md")
+        submit_btn.click(
+            fn=math_chat_bot,
+            inputs=[*input_image, input_sketchpad, input_text, state],
+            outputs=output_md)
+demo.launch()