Explainable-Vision-Language-Model

Running on Zero

App Files Files Community

khang119966 commited on 22 days ago

Commit

42bde0f

verified ·

1 Parent(s): ba17d2e

Update app.py

Browse files

Files changed (1) hide show

app.py +309 -1

app.py CHANGED Viewed

@@ -133,7 +133,312 @@ def load_image(image_file, input_size=448, max_num=12, target_aspect_ratio=False
         return pixel_values, target_aspect_ratio
     else:
         return pixel_values
 model = AutoModel.from_pretrained(
     "khang119966/Vintern-1B-v3_5-explainableAI",
     torch_dtype=torch.bfloat16,
@@ -150,6 +455,9 @@ def generate_video(image, prompt, max_tokens):
     response, query = model.chat(tokenizer, pixel_values, '<image>\n'+prompt, generation_config, return_history=False, \
                             attention_visualize=True,last_visualize_layers=7,raw_image_path=test_image,target_aspect_ratio=target_aspect_ratio)
     print(response)
     return "path_to_generated_video.mp4"
 with gr.Blocks() as demo:
@@ -157,7 +465,7 @@ with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column():
-            image = gr.Image(label="Upload your image")
             prompt = gr.Textbox(label="Describe your prompt", value="List all the text." )
             max_tokens = gr.Slider(label="Max token output (⚠️ Choose <100 for faster response)", minimum=1, maximum=512, value=50)
             btn = gr.Button("Attenion Video")

         return pixel_values, target_aspect_ratio
     else:
         return pixel_values
+def visualize_attention_hiddenstate(attention_tensor, head=None, start_img_token_index=0, end_img_token_index=0, target_aspect_ratio=(0,0)):
+    """Vẽ heatmap của attention scores từ trung bình 8 layer cuối và trả về top 5 token có attention cao nhất."""
+    last_8_layers = attention_tensor[-8:]  # Lấy 8 layer cuối
+    averaged_layer = np.mean(last_8_layers,axis=0)  # Trung bình 8 layer cuối
+    if head is None:
+        averaged_attention = averaged_layer.mean(axis=1).squeeze()  # Trung bình qua các head
+    else:
+        averaged_attention = averaged_layer[:, head, :, :].squeeze()  # Chọn head cụ thể
+    heat_maps = []
+    top_5_tokens = []
+    for i in range(len(averaged_attention)):  # Duyệt qua các beam
+        h_target_aspect_ratio = target_aspect_ratio[1] if target_aspect_ratio[1] != 0 else 1
+        w_target_aspect_ratio = target_aspect_ratio[0] if target_aspect_ratio[0] != 0 else 1
+        img_atten_score = averaged_attention[i].reshape(-1)[start_img_token_index:end_img_token_index]
+        # Lấy index của 5 token có attention cao nhất
+        top_5_indices = np.argsort(img_atten_score)[-5:][::-1]  # Sắp xếp giảm dần
+        top_5_values = img_atten_score[top_5_indices]
+        # top_5_tokens.append(list(zip(top_5_indices + start_img_token_index, top_5_values)))
+        top_5_tokens.append(list(top_5_indices + start_img_token_index))
+        # Reshape lại attention để vẽ heatmap
+        img_atten_score = img_atten_score.reshape(h_target_aspect_ratio, w_target_aspect_ratio, 16, 16)
+        img_atten_score = np.transpose(img_atten_score, (0, 2, 1, 3)).reshape(h_target_aspect_ratio * 16, w_target_aspect_ratio * 16)
+        img_atten_score = np.power(img_atten_score, 0.9)
+        heat_maps.append(img_atten_score)
+    return heat_maps, top_5_tokens
+def generate_next_token_table_image(model, tokenizer, response, index_focus):
+    next_token_table = []
+    for layer_index in range(len(response.hidden_states[index_focus])):
+        h_out = model.language_model.lm_head(
+            model.language_model.model.norm(response.hidden_states[index_focus][layer_index][0])
+        )
+        h_out = torch.softmax(h_out, -1)
+        top_tokens = []
+        for token_index in h_out.argsort(descending=True)[0, :3]:  # Top 3
+            token_str = tokenizer.decode(token_index)
+            prob = float(h_out[0, int(token_index)])
+            top_tokens.append((token_str, prob))
+        next_token_table.append((layer_index, top_tokens))
+    next_token_table = next_token_table[::-1]
+    html_rows = ""
+    last_layer_index = len(next_token_table) - 1
+    for i, (layer_index, tokens) in enumerate(next_token_table):
+        row = f"<tr><td style='font-weight: bold'>Layer {layer_index}</td>"
+        # For the first column (Top 1)
+        token_str, prob = tokens[0]
+        # If this is the last layer in the table, make the text blue
+        if layer_index == last_layer_index:
+            row += f"<td><span style='color: red; font-weight: bold'>{token_str}</span> ({prob:.2%})</td>"
+        else:
+            row += f"<td><span style='color: blue; font-weight: bold'>{token_str}</span> ({prob:.2%})</td>"
+        # For the other columns, keep normal formatting
+        for token_str, prob in tokens[1:]:
+            row += f"<td>{token_str} ({prob:.2%})</td>"
+        row += "</tr>"
+        html_rows += row
+    html_code = f'''
+    <html>
+      <head>
+        <meta charset="utf-8">
+        <style>
+          table {{
+            font-family: 'Noto Sans';
+            font-size: 12px;
+            border-collapse: collapse;
+            table-layout: fixed;
+            width: 100%;
+          }}
+          th, td {{
+            border: 1px solid black;
+            padding: 8px;
+            width: 150px;
+            height: 30px;
+            overflow: hidden;
+            text-overflow: ellipsis;
+            white-space: nowrap;
+            text-align: center;
+          }}
+          th.layer {{
+            width: 100px;
+          }}
+          th.title {{
+            font-size: 14px;
+            padding: 10px;
+            height: auto;
+            white-space: normal;
+            overflow: visible;
+          }}
+        </style>
+      </head>
+      <body style="background-color: white;">
+        <table>
+          <tr>
+            <th colspan="4" class="title">
+              Top hidden tokens per layer for the Prediction
+            </th>
+          </tr>
+          <tr>
+            <th class="layer">Layer ⬆️</th>
+            <th>Top 1</th>
+            <th>Top 2</th>
+            <th>Top 3</th>
+          </tr>
+          {html_rows}
+        </table>
+      </body>
+    </html>
+    '''
+    with tempfile.TemporaryDirectory() as tmpdir:
+        hti = Html2Image(output_path=tmpdir)
+        hti.browser_flags = [
+    "--headless=new",      # ← Dùng chế độ headless mới
+    "--disable-gpu",       # ← Tắt GPU
+    "--disable-software-rasterizer",  # ← Tránh dùng fallback GPU software
+    "--no-sandbox",        # ← Tránh lỗi sandbox đa luồng
+]
+        filename = str(uuid.uuid4())+".png"
+        # filename = 'next_token_table.png'
+        hti.screenshot(html_str=html_code, save_as=filename, size=(500, 1000))
+        img_path = os.path.join(tmpdir, filename)
+        img_cv2 = cv2.imread(img_path)[:,:,::-1]
+        os.remove(img_path)
+    return img_cv2
+def adjust_overlay(overlay, text_img):
+    h_o, w_o = overlay.shape[:2]
+    h_t, w_t = text_img.shape[:2]
+    if h_o > w_o:  # Overlay là ảnh đứng
+        # Resize overlay sao cho h = h_t, giữ nguyên tỷ lệ
+        new_h = h_t
+        new_w = int(w_o * (new_h / h_o))
+        overlay_resized = cv2.resize(overlay, (new_w, new_h))
+    else:  # Overlay là ảnh ngang
+        # Giữ nguyên overlay, nhưng nếu h < h_t thì thêm padding trắng
+        overlay_resized = overlay.copy()
+    # Thêm padding trắng nếu overlay có h < h_t
+    if overlay_resized.shape[0] < h_t:
+        pad_h = h_t - overlay_resized.shape[0]
+        padding = np.ones((pad_h, overlay_resized.shape[1], 3), dtype=np.uint8) * 255
+        overlay_resized = np.vstack((overlay_resized, padding))  # Padding vào dưới
+    # Đảm bảo overlay có cùng chiều cao với text_img
+    if overlay_resized.shape[0] != h_t:
+        overlay_resized = cv2.resize(overlay_resized, (overlay_resized.shape[1], h_t))
+    return overlay_resized
+def generate_text_image_with_html2image(old_text, input_token, new_token, image_width=400, min_height=1000, font_size=16):
+    full_text = old_text + f"<span style='color:blue; font-weight:bold'>[{input_token}]</span>"+ "→" + f"<span style='color:red; font-weight:bold'>[{new_token}]</span>"
+    # Thay \n bằng thẻ HTML <br> để xuống dòng
+    full_text = full_text.replace('\n', '<br>')
+    html_code = f'''
+    <html>
+    <head>
+        <meta charset="utf-8">
+    </head>
+    <body style="font-family: 'DejaVu Sans', sans-serif; font-size: {font_size}px; width: {image_width}px; min-height: {min_height}px; padding: 10px; background-color: white; line-height: 1.4;">
+        {full_text}
+    </body>
+    </html>
+    '''
+    save_path = str(uuid.uuid4())+".png"
+    hti = Html2Image(output_path='.')
+    hti.browser_flags = [
+    "--headless=new",      # ← Dùng chế độ headless mới
+    "--disable-gpu",       # ← Tắt GPU
+    "--disable-software-rasterizer",  # ← Tránh dùng fallback GPU software
+    "--no-sandbox",        # ← Tránh lỗi sandbox đa luồng
+]
+    hti.screenshot(html_str=html_code, save_as=save_path, size=(image_width, min_height))
+    text_img = cv2.imread(save_path)
+    text_img = cv2.cvtColor(text_img, cv2.COLOR_BGR2RGB)
+    os.remove(save_path)
+    return text_img
+def extract_next_token_table_data(model, tokenizer, response, index_focus):
+    next_token_table = []
+    for layer_index in range(len(response.hidden_states[index_focus])):
+        h_out = model.language_model.lm_head(
+            model.language_model.model.norm(response.hidden_states[index_focus][layer_index][0])
+        )
+        h_out = torch.softmax(h_out, -1)
+        top_tokens = []
+        for token_index in h_out.argsort(descending=True)[0, :3]:  # Top 3
+            token_str = tokenizer.decode(token_index)
+            prob = float(h_out[0, int(token_index)])
+            top_tokens.append((token_str, prob))
+        next_token_table.append((layer_index, top_tokens))
+    next_token_table = next_token_table[::-1]
+    return next_token_table
+def render_next_token_table_image(table_data, predict_token):
+    import tempfile, uuid, os
+    from html2image import Html2Image
+    import cv2
+    html_rows = ""
+    last_layer_index = len(table_data)
+    for layer_index, tokens in table_data:
+        row = f"<tr><td style='font-weight: bold'>Layer {layer_index+1}</td>"
+        token_str, prob = tokens[0]
+        if token_str == predict_token:
+            style = "color: red; font-weight: bold"
+        else:
+            style = "color: blue; font-weight: bold"
+        row += f"<td><span style='{style}'>{token_str}</span> ({prob:.2%})</td>"
+        for token_str, prob in tokens[1:]:
+            row += f"<td>{token_str} ({prob:.2%})</td>"
+        row += "</tr>"
+        html_rows += row
+    html_code = f'''
+    <html>
+      <head>
+        <meta charset="utf-8">
+        <style>
+          table {{
+            font-family: 'Noto Sans';
+            font-size: 12px;
+            border-collapse: collapse;
+            table-layout: fixed;
+            width: 100%;
+          }}
+          th, td {{
+            border: 1px solid black;
+            padding: 8px;
+            width: 150px;
+            height: 30px;
+            overflow: hidden;
+            text-overflow: ellipsis;
+            white-space: nowrap;
+            text-align: center;
+          }}
+          th.layer {{
+            width: 100px;
+          }}
+          th.title {{
+            font-size: 14px;
+            padding: 10px;
+            height: auto;
+            white-space: normal;
+            overflow: visible;
+          }}
+        </style>
+      </head>
+      <body style="background-color: white;">
+        <table>
+          <tr>
+            <th colspan="4" class="title">
+              Hidden states per Transformer layer (LLM) for Prediction
+            </th>
+          </tr>
+          <tr>
+            <th class="layer">Layer ⬆️</th>
+            <th>Top 1</th>
+            <th>Top 2</th>
+            <th>Top 3</th>
+          </tr>
+          {html_rows}
+        </table>
+      </body>
+    </html>
+    '''
+    with tempfile.TemporaryDirectory() as tmpdir:
+        hti = Html2Image(output_path=tmpdir)
+        hti.browser_flags = [
+            "--headless=new",
+            "--disable-gpu",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+        ]
+        filename = str(uuid.uuid4()) + ".png"
+        hti.screenshot(html_str=html_code, save_as=filename, size=(500, 1000))
+        img_path = os.path.join(tmpdir, filename)
+        img_cv2 = cv2.imread(img_path)[:, :, ::-1]
+        os.remove(img_path)
+    return img_cv2
 model = AutoModel.from_pretrained(
     "khang119966/Vintern-1B-v3_5-explainableAI",
     torch_dtype=torch.bfloat16,
     response, query = model.chat(tokenizer, pixel_values, '<image>\n'+prompt, generation_config, return_history=False, \
                             attention_visualize=True,last_visualize_layers=7,raw_image_path=test_image,target_aspect_ratio=target_aspect_ratio)
     print(response)
+    generation_output = response
+    raw_image_path = image
     return "path_to_generated_video.mp4"
 with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column():
+            image = gr.Image(label="Upload your image", type = 'filepath')
             prompt = gr.Textbox(label="Describe your prompt", value="List all the text." )
             max_tokens = gr.Slider(label="Max token output (⚠️ Choose <100 for faster response)", minimum=1, maximum=512, value=50)
             btn = gr.Button("Attenion Video")