runninglsy commited on
Commit
b816677
·
1 Parent(s): a468591
.gitattributes CHANGED
@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ *.png filter=lfs diff=lfs merge=lfs -text
37
+ *.jpg filter=lfs diff=lfs merge=lfs -text
38
+ *.jpeg filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,14 +1,14 @@
1
  ---
2
- title: Ovis2 1B
3
- emoji: 👁
4
- colorFrom: green
5
- colorTo: red
6
  sdk: gradio
7
- sdk_version: 5.16.0
8
  app_file: app.py
9
  pinned: false
10
  license: apache-2.0
11
- short_description: Ovis2-1B
12
  ---
13
 
14
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Ovis2 16B
3
+ emoji: 🦫
4
+ colorFrom: yellow
5
+ colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 5.1.0
8
  app_file: app.py
9
  pinned: false
10
  license: apache-2.0
11
+ short_description: Ovis2-16B
12
  ---
13
 
14
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import subprocess
2
+ subprocess.run('pip install flash-attn==2.7.0.post2 --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
3
+
4
+ import spaces
5
+ import os
6
+ import re
7
+ import logging
8
+ from typing import List, Any
9
+ from threading import Thread
10
+
11
+ import torch
12
+ import gradio as gr
13
+ from transformers import AutoModelForCausalLM, TextIteratorStreamer
14
+
15
+ model_name = 'AIDC-AI/Ovis2-1B'
16
+ use_thread = False
17
+
18
+ # load model
19
+ model = AutoModelForCausalLM.from_pretrained(model_name,
20
+ torch_dtype=torch.bfloat16,
21
+ multimodal_max_length=8192,
22
+ trust_remote_code=True).to(device='cuda')
23
+ text_tokenizer = model.get_text_tokenizer()
24
+ visual_tokenizer = model.get_visual_tokenizer()
25
+ streamer = TextIteratorStreamer(text_tokenizer, skip_prompt=True, skip_special_tokens=True)
26
+ image_placeholder = '<image>'
27
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
28
+
29
+ logging.basicConfig(level=logging.INFO)
30
+ logger = logging.getLogger(__name__)
31
+
32
+ def initialize_gen_kwargs():
33
+ return {
34
+ "max_new_tokens": 1536,
35
+ "do_sample": False,
36
+ "top_p": None,
37
+ "top_k": None,
38
+ "temperature": None,
39
+ "repetition_penalty": 1.05,
40
+ "eos_token_id": model.generation_config.eos_token_id,
41
+ "pad_token_id": text_tokenizer.pad_token_id,
42
+ "use_cache": True
43
+ }
44
+
45
+ def submit_chat(chatbot, text_input):
46
+ response = ''
47
+ chatbot.append((text_input, response))
48
+ return chatbot ,''
49
+
50
+ @spaces.GPU
51
+ def ovis_chat(chatbot: List[List[str]], image_input: Any):
52
+ conversations, model_inputs = prepare_inputs(chatbot, image_input)
53
+ gen_kwargs = initialize_gen_kwargs()
54
+
55
+ with torch.inference_mode():
56
+ generate_func = lambda: model.generate(**model_inputs, **gen_kwargs, streamer=streamer)
57
+
58
+ if use_thread:
59
+ thread = Thread(target=generate_func)
60
+ thread.start()
61
+ else:
62
+ generate_func()
63
+
64
+ response = ""
65
+ for new_text in streamer:
66
+ response += new_text
67
+ chatbot[-1][1] = response
68
+ yield chatbot
69
+
70
+ if use_thread:
71
+ thread.join()
72
+
73
+ log_conversation(chatbot)
74
+
75
+
76
+ def prepare_inputs(chatbot: List[List[str]], image_input: Any):
77
+ # conversations = [{
78
+ # "from": "system",
79
+ # "value": "You are a helpful assistant, and your task is to provide reliable and structured responses to users."
80
+ # }]
81
+ conversations= []
82
+
83
+ for query, response in chatbot[:-1]:
84
+ conversations.extend([
85
+ {"from": "human", "value": query},
86
+ {"from": "gpt", "value": response}
87
+ ])
88
+
89
+ last_query = chatbot[-1][0].replace(image_placeholder, '')
90
+ conversations.append({"from": "human", "value": last_query})
91
+
92
+ if image_input is not None:
93
+ for conv in conversations:
94
+ if conv["from"] == "human":
95
+ conv["value"] = f'{image_placeholder}\n{conv["value"]}'
96
+ break
97
+
98
+ logger.info(conversations)
99
+
100
+ prompt, input_ids, pixel_values = model.preprocess_inputs(conversations, [image_input], max_partition=16)
101
+ attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id)
102
+
103
+ model_inputs = {
104
+ "inputs": input_ids.unsqueeze(0).to(device=model.device),
105
+ "attention_mask": attention_mask.unsqueeze(0).to(device=model.device),
106
+ "pixel_values": [pixel_values.to(dtype=visual_tokenizer.dtype, device=visual_tokenizer.device)] if image_input is not None else [None]
107
+ }
108
+
109
+ return conversations, model_inputs
110
+
111
+ def log_conversation(chatbot):
112
+ logger.info("[OVIS_CONV_START]")
113
+ [print(f'Q{i}:\n {request}\nA{i}:\n {answer}') for i, (request, answer) in enumerate(chatbot, 1)]
114
+ logger.info("[OVIS_CONV_END]")
115
+
116
+ def clear_chat():
117
+ return [], None, ""
118
+
119
+ with open(f"{cur_dir}/resource/logo.svg", "r", encoding="utf-8") as svg_file:
120
+ svg_content = svg_file.read()
121
+ font_size = "2.5em"
122
+ svg_content = re.sub(r'(<svg[^>]*)(>)', rf'\1 height="{font_size}" style="vertical-align: middle; display: inline-block;"\2', svg_content)
123
+ html = f"""
124
+ <p align="center" style="font-size: {font_size}; line-height: 1;">
125
+ <span style="display: inline-block; vertical-align: middle;">{svg_content}</span>
126
+ <span style="display: inline-block; vertical-align: middle;">{model_name.split('/')[-1]}</span>
127
+ </p>
128
+ <center><font size=3><b>Ovis</b> has been open-sourced on <a href='https://huggingface.co/{model_name}'>😊 Huggingface</a> and <a href='https://github.com/AIDC-AI/Ovis'>🌟 GitHub</a>. If you find Ovis useful, a like❤️ or a star🌟 would be appreciated.</font></center>
129
+ """
130
+
131
+ latex_delimiters_set = [{
132
+ "left": "\\(",
133
+ "right": "\\)",
134
+ "display": False
135
+ }, {
136
+ "left": "\\begin{equation}",
137
+ "right": "\\end{equation}",
138
+ "display": True
139
+ }, {
140
+ "left": "\\begin{align}",
141
+ "right": "\\end{align}",
142
+ "display": True
143
+ }, {
144
+ "left": "\\begin{alignat}",
145
+ "right": "\\end{alignat}",
146
+ "display": True
147
+ }, {
148
+ "left": "\\begin{gather}",
149
+ "right": "\\end{gather}",
150
+ "display": True
151
+ }, {
152
+ "left": "\\begin{CD}",
153
+ "right": "\\end{CD}",
154
+ "display": True
155
+ }, {
156
+ "left": "\\[",
157
+ "right": "\\]",
158
+ "display": True
159
+ }]
160
+
161
+ text_input = gr.Textbox(label="prompt", placeholder="Enter your text here...", lines=1, container=False)
162
+ with gr.Blocks(title=model_name.split('/')[-1], theme=gr.themes.Ocean()) as demo:
163
+ gr.HTML(html)
164
+ with gr.Row():
165
+ with gr.Column(scale=3):
166
+ image_input = gr.Image(label="image", height=350, type="pil")
167
+ gr.Examples(
168
+ examples=[
169
+ [f"{cur_dir}/examples/ovis2_caption0.jpg", "Describe the image."],
170
+ [f"{cur_dir}/examples/ovis2_ocr1.jpg", "Recognize text in images."],
171
+ [f"{cur_dir}/examples/ovsi2_know0.png", "What is the animal in the picture?"],
172
+ ],
173
+ inputs=[image_input, text_input]
174
+ )
175
+ with gr.Column(scale=7):
176
+ chatbot = gr.Chatbot(label="Ovis", layout="panel", height=600, show_copy_button=True, latex_delimiters=latex_delimiters_set)
177
+ text_input.render()
178
+ with gr.Row():
179
+ send_btn = gr.Button("Send", variant="primary")
180
+ clear_btn = gr.Button("Clear", variant="secondary")
181
+
182
+ send_click_event = send_btn.click(submit_chat, [chatbot, text_input], [chatbot, text_input]).then(ovis_chat,[chatbot, image_input],chatbot)
183
+ submit_event = text_input.submit(submit_chat, [chatbot, text_input], [chatbot, text_input]).then(ovis_chat,[chatbot, image_input],chatbot)
184
+ clear_btn.click(clear_chat, outputs=[chatbot, image_input, text_input])
185
+
186
+ demo.launch()
examples/ovis2_caption0.jpg ADDED

Git LFS Details

  • SHA256: ddd7939e7b7be2813bd9e11ac21c016b05b10332e5e519b7500c10aca7f32c88
  • Pointer size: 131 Bytes
  • Size of remote file: 437 kB
examples/ovis2_figure0.png ADDED

Git LFS Details

  • SHA256: 80bebf1106831041eaa9baef86d12d443360d5f4e5dd37795d841658853b44fc
  • Pointer size: 132 Bytes
  • Size of remote file: 2.84 MB
examples/ovis2_figure1.png ADDED

Git LFS Details

  • SHA256: af401830ffa31eac748766c49cc678124f859aa5336c38c94b3586fda0e6240c
  • Pointer size: 131 Bytes
  • Size of remote file: 278 kB
examples/ovis2_math0.jpg ADDED

Git LFS Details

  • SHA256: e9feb598f783b0103888fa6db1dea23045e9245d8417895623f8408b783c062e
  • Pointer size: 129 Bytes
  • Size of remote file: 7.46 kB
examples/ovis2_math1.jpg ADDED

Git LFS Details

  • SHA256: d8a7dc778bae422f40e37ecd6e23e99a08be5d1c81b5d92530d4572bc6d8e2b4
  • Pointer size: 129 Bytes
  • Size of remote file: 6.39 kB
examples/ovis2_multi0.jpg ADDED

Git LFS Details

  • SHA256: 66f1f86d24b0f334f039165ebd1ec3e83cefcf7b8bea87e9ec2d42a09c1f84e5
  • Pointer size: 132 Bytes
  • Size of remote file: 3.41 MB
examples/ovis2_ocr0.jpeg ADDED

Git LFS Details

  • SHA256: f814f1c1c0899bde9e6469fd09d565df9420c4b5ee14e19be097296df19a19f7
  • Pointer size: 131 Bytes
  • Size of remote file: 145 kB
examples/ovis2_ocr1.jpg ADDED

Git LFS Details

  • SHA256: 0b6bf32049e611197aded3dfb7d63e83054412706e2cf24de31561566f4108d2
  • Pointer size: 131 Bytes
  • Size of remote file: 137 kB
examples/ovsi2_know0.png ADDED

Git LFS Details

  • SHA256: ed7a95a6bd29ef5f9e3727f44cbb0c475a52d7d2a7d397214687b34d7f14812f
  • Pointer size: 131 Bytes
  • Size of remote file: 318 kB
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ numpy==1.25.0
2
+ torch==2.4.0
3
+ transformers==4.46.2
4
+ pillow==10.3.0
resource/logo.svg ADDED