add alignment and specify encoder

app.py

@@ -1,7 +1,337 @@
+import os
+import gc
 import gradio as gr
+import torch
+import random
+import logging
+import openai
+from openai import OpenAI
+from vecalign.plan2align import translate_text, external_find_best_translation
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from trl import AutoModelForCausalLMWithValueHead
+from huggingface_hub import login
+import spacy
+import subprocess
+import pkg_resources
+import sys
 
-def greet(name):
-    return "Hello " + name + "!!"
+laser_token = os.environ.get("align_enc")
+laser_path = snapshot_download(repo_id="KuangDW/laser", use_auth_token=hf_token)
+os.environ["LASER"] = laser_path
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+def check_and_install(package, required_version):
+    try:
+        dist = pkg_resources.get_distribution(package)
+        installed_version = dist.version
+        if installed_version != required_version:
+            print(f"[{package}] already installed {installed_version}. Required version  {required_version}，re-install...")
+            subprocess.check_call([sys.executable, "-m", "pip", "install", f"{package}=={required_version}", "--force-reinstall"])
+        else:
+            print(f"[{package}] required version  {required_version} finished")
+    except pkg_resources.DistributionNotFound:
+        print(f"[{package}] not found, install: {required_version}...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", f"{package}=={required_version}"])
+packages = {
+    "pip": "24.0",
+    "fairseq": "0.12.2"
+}
+for package, version in packages.items():
+    check_and_install(package, version)
+
+
+models = ["en_core_web_sm", "ru_core_news_sm", "de_core_news_sm", 
+          "ja_core_news_sm", "ko_core_news_sm", "es_core_news_sm"]
+for model in models:
+    try:
+        spacy.load(model)
+    except OSError:
+        from spacy.cli import download
+        download(model)
+try:
+    spacy.load("zh_core_web_sm")
+except OSError:
+    from spacy.cli import download
+    download("zh_core_web_sm")
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "numpy==1.24.0", "--force-reinstall"])
+
+# ---------- deepinfra translation ----------
+openai = OpenAI(
+    api_key="",
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+def generate_translation(system_prompt, prompt):
+    response = openai.chat.completions.create(
+        model="meta-llama/Meta-Llama-3.1-8B-Instruct",
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": prompt}
+        ]
+    )
+    translation = response.choices[0].message.content.strip()
+    return translation
+
+def check_token_length(text, max_tokens=1024):
+    return len(text) <= max_tokens
+
+import uuid
+def get_user_session(state):
+    if not state.get("session_id"):
+        state["session_id"] = uuid.uuid4().hex
+    return state["session_id"]
+
+
+# ---------- Translation Function ----------
+
+def mpc_initial_translate(source_sentence, src_language, tgt_language):
+    system_prompts = [
+        "You are a meticulous translator. Provide a literal, word-for-word translation that preserves the structure and meaning of each individual word.",
+        "You are a professional translator. Deliver a clear, formal, and precise translation that faithfully conveys the original meaning.",
+        "You are a creative and expressive translator. Render the text in a vivid and imaginative way, as if narrating a captivating story."
+    ]
+    translations = []
+    for prompt_style in system_prompts:
+        prompt = f"### Translate this from {src_language} to {tgt_language} and only output the result."
+        prompt += f"\n### {src_language}:\n {source_sentence}"
+        prompt += f"\n### {tgt_language}:\n"
+        translation = generate_translation(prompt_style, prompt)
+        translations.append(translation)
+    
+    print("mpc_initial_translate")
+    print(translations)
+    return translations
+
+def mpc_improved_translate(source_sentence, current_translation, src_language, tgt_language):
+    system_prompts = [
+        "You are a meticulous translator. Please improve the following translation by ensuring it is a literal and structurally precise version.",
+        "You are a professional translator. Please refine the provided translation to be clear, formal, and accurate.",
+        "You are a creative translator. Please enhance the translation so that it is vivid, natural, and engaging."
+    ]
+    translations = []
+    for prompt_style in system_prompts:
+        prompt = (f"Source ({src_language}): {source_sentence}\n"
+                  f"Current Translation ({tgt_language}): {current_translation}\n"
+                  f"Please provide an improved translation into {tgt_language} and only output the result:")
+        translation = generate_translation(prompt_style, prompt)
+        translations.append(translation)
+    
+    print("mpc_improved_translate")
+    print(translations)
+    return translations
+
+def basic_translate(source_sentence, src_language, tgt_language):
+    system_prompts = ["You are a helpful translator and only output the result."]
+    translations = []
+    for prompt_style in system_prompts:
+        prompt = f"### Translate this from {src_language} to {tgt_language}."
+        prompt += f"\n### {src_language}:\n {source_sentence}"
+        prompt += f"\n### {tgt_language}:\n"
+        translation = generate_translation(prompt_style, prompt)
+        translations.append(translation)
+    return translations 
+
+def plan2align_translate_text(text, session_id, src_language, task_language, max_iterations_value, threshold_value, good_ref_contexts_num_value, reward_model_type):
+    result = translate_text(
+        text,
+        src_language=src_language,
+        task_language=task_language,
+        max_iterations_value=max_iterations_value,
+        threshold_value=threshold_value,
+        good_ref_contexts_num_value=good_ref_contexts_num_value,
+        reward_model_type=reward_model_type,
+        session_id=session_id
+    )
+    _, score = evaluate_candidates(text, [result], task_language, session_id)
+    return result, score
+
+def evaluate_candidates(source, candidates, language, session_id):
+    evals = [(source, candidates)]
+    best_translations = external_find_best_translation(evals, language, session_id)
+    best_candidate, best_score = best_translations[0]
+    return best_candidate, best_score
+
+def original_translation(text, src_language, target_language, session_id):
+    cand_list = basic_translate(text, src_language, target_language)
+    best, score = evaluate_candidates(text, cand_list, target_language, session_id)
+    if cand_list:
+        return best, score
+    return "", 0
+
+def best_of_n_translation(text, src_language, target_language, n, session_id):
+    if not check_token_length(text, 2048):
+         return "Warning: Input text exceeds 2048 tokens.", None, ""
+    candidates = []
+    for i in range(n):
+        cand_list = basic_translate(text, src_language, target_language)
+        if cand_list:
+            candidates.append(cand_list[0])
+    best, score = evaluate_candidates(text, candidates, target_language, session_id)
+    print("best_of_n evaluate_candidates results:")
+    print(best, score)
+    return best, score
+
+def mpc_translation(text, src_language, target_language, iterations, session_id):
+    if not check_token_length(text, 2048):
+         return "Warning: Input text exceeds 2048 tokens.", None, ""
+    current_trans = ""
+    best_score = None
+    for i in range(iterations):
+        if i == 0:
+            cand_list = mpc_initial_translate(text, src_language, target_language)
+        else:
+            cand_list = mpc_improved_translate(text, current_trans, src_language, target_language)        
+        
+        best, score = evaluate_candidates(text, cand_list, target_language, session_id)
+        print("mpc evaluate_candidates results:")
+        print(best, score)
+        current_trans = best
+        best_score = score
+    return current_trans, best_score
+
+# ---------- Gradio 主流程函數 ----------
+
+def process_text(text, src_language, target_language, max_iterations_value, threshold_value,
+                 good_ref_contexts_num_value, translation_methods, state):
+    session_id = get_user_session(state)
+
+    """
+    傳入中文文本與目標語言，依序產生四種翻譯結果：
+      1. 原始翻譯
+      2. Plan2Align 翻譯
+      3. Best-of-N 翻譯
+      4. MPC 翻譯
+    """
+
+    # 初始化各輸出內容
+    orig_output = ""
+    plan2align_output = ""
+    best_of_n_output = ""
+    mpc_output = ""
+
+
+    if "Original" in translation_methods:
+        orig, best_score = original_translation(text, src_language, target_language, session_id)
+        orig_output = f"{orig}\n\nScore: {best_score:.2f}"
+    if "Plan2Align" in translation_methods:
+        plan2align_trans, best_score = plan2align_translate_text(
+            text, session_id, src_language, target_language,
+            max_iterations_value, threshold_value, good_ref_contexts_num_value, "metricx"
+        )
+        plan2align_output = f"{plan2align_trans}\n\nScore: {best_score:.2f}"
+    if "Best-of-N" in translation_methods:
+        best_candidate, best_score = best_of_n_translation(text, src_language, target_language,
+                                                            max_iterations_value, session_id)
+        best_of_n_output = f"{best_candidate}\n\nScore: {best_score:.2f}"
+    if "MPC" in translation_methods:
+        mpc_candidate, mpc_score = mpc_translation(text, src_language, target_language,
+                                                   max_iterations_value, session_id)
+        mpc_output = f"{mpc_candidate}\n\nScore: {mpc_score:.2f}"
+    
+    return orig_output, plan2align_output, best_of_n_output, mpc_output
+
+# ---------- Gradio ----------
+target_languages = ["Chinese", "English", "Russian", "German", "Japanese", "Korean"]
+src_languages = ["Chinese", "English", "Russian", "German", "Japanese", "Korean"]
+
+with gr.Blocks(title="Test-Time Machine Translation with Plan2Align") as demo:
+    state = gr.State({})
+
+    gr.Markdown("# Translation Demo: Multiple Translation Methods")
+    gr.Markdown("請選擇要執行的翻譯方法（可多選或全選）：")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            source_text = gr.Textbox(
+                label="Source Text",
+                placeholder="請輸入文本...",
+                lines=5
+            )
+            src_language_input = gr.Dropdown(
+                choices=src_languages,
+                value="Chinese",
+                label="Source Language"
+            )
+            task_language_input = gr.Dropdown(
+                choices=target_languages,
+                value="English",
+                label="Task (Target) Language"
+            )
+            max_iterations_input = gr.Number(label="Max Iterations", value=6)
+            threshold_input = gr.Number(label="Threshold", value=0.7)
+            good_ref_contexts_num_input = gr.Number(label="Good Ref Contexts Num", value=5)
+            translation_methods_input = gr.CheckboxGroup(
+                choices=["Original", "Plan2Align", "Best-of-N", "MPC"],
+                value=["Original", "Plan2Align"],
+                label="Translation Methods"
+            )
+            translate_button = gr.Button("Translate")
+        with gr.Column(scale=2):
+            original_output = gr.Textbox(
+                label="Original Translation",
+                lines=5,
+                interactive=False
+            )
+            plan2align_output = gr.Textbox(
+                label="Plan2Align Translation",
+                lines=5,
+                interactive=False
+            )
+            best_of_n_output = gr.Textbox(
+                label="Best-of-N Translation",
+                lines=5,
+                interactive=False
+            )
+            mpc_output = gr.Textbox(
+                label="MPC Translation",
+                lines=5,
+                interactive=False
+            )
+    
+    translate_button.click(
+        fn=process_text,
+        inputs=[
+            source_text,
+            src_language_input,
+            task_language_input,
+            max_iterations_input,
+            threshold_input,
+            good_ref_contexts_num_input,
+            translation_methods_input,
+            state
+        ],
+        outputs=[original_output, plan2align_output, best_of_n_output, mpc_output]
+    )
+    
+    gr.Examples(
+        examples=[
+            ["夜市文化豐富多彩，從士林夜市到饒河街夜市，提供各種美食、遊戲和購物體驗，吸引了無數遊客。", "Chinese", "English", 2, 0.7, 1],
+            ["台北101曾經是世界最高的建築物，它不僅是台灣的地標，也象徵著經濟成就和創新精神。", "Chinese", "Russian", 2, 0.7, 1],
+            ["阿里山日出和森林鐵路是台灣最著名的自然景觀之一，每年吸引數十萬遊客前來欣賞雲海和壯麗的日出。", "Chinese", "German", 2, 0.7, 1],
+            ["珍珠奶茶，這款源自台灣的獨特飲品，不僅在台灣本地深受喜愛，更以其獨特的風味和口感，在全球掀起了一股熱潮，成為了一種跨越文化、風靡全球的時尚飲品。", "Chinese", "Japanese", 3, 0.7, 3],
+            ["原住民文化如同一片深邃的星空，閃爍著無數璀璨的傳統與藝術光芒。他們的歌舞，是與祖靈對話的旋律，是與自然共鳴的節奏，每一個舞步、每一聲吟唱，都承載著古老的傳說與智慧。編織，是他們巧手下的詩篇，一絲一線，交織出生命的紋理，也編織出對土地的熱愛與敬畏。木雕，則是他們與自然對話的雕塑，每一刀、每一鑿，都刻畫著對萬物的觀察與敬意，也雕琢出對祖先的追憶與傳承。", "Chinese", "Korean", 5, 0.7, 5]
+        ],
+        inputs=[
+            source_text,
+            src_language_input,
+            task_language_input,
+            max_iterations_input,
+            threshold_input,
+            good_ref_contexts_num_input,
+            state
+        ],
+        outputs=[original_output, plan2align_output, best_of_n_output, mpc_output],
+        fn=process_text
+    )
+    
+    gr.Markdown("## How It Works")
+    gr.Markdown("""
+    1. **Original Translation:** 利用固定提示生成候選，直接取首個候選作為原始翻譯。
+    2. **Plan2Align Translation:** 採用 context alignment 和 self-rewriting 策略進行翻譯，適合長文翻譯。
+    3. **Best-of-N Translation:** 重複生成多次候選，評分選出最佳翻譯，適合短文翻譯。
+    4. **MPC Translation:** 以迭代改善策略，每輪生成候選後評分，並將最佳翻譯作為下一輪輸入，適合短文翻譯。
+    
+    若輸入文本超過 1024 tokens，Best-of-N 與 MPC 方法會回傳警告訊息。
+    """)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,36 @@
+pip==24.0
+botok==0.9.0
+Cython==3.0.11
+datasets==3.4.1
+emoji==2.14.1
+Flask==3.1.0
+func_argparse==1.1.1
+gradio==5.25.0
+indicnlp==0.0.1
+laonlp==1.2.0
+mcerp==0.12
+openai==1.73.0
+pandas==2.2.3
+pytest==8.3.5
+Requests==2.32.3
+sacremoses==0.1.1
+safetensors==0.5.3
+scipy==1.15.2
+sentence_splitter==1.4
+sentence_transformers==4.0.2
+setuptools==69.0.3
+spacy==3.8.3
+stopes==2.2.1
+tabulate==0.9.0
+tiktoken==0.8.0
+torch==2.1.2
+tqdm==4.67.1
+transformers>=4.41.0,<5.0.0
+transliterate==1.10.2
+trl==0.16.1
+unicategories==0.1.2
+xxhash==3.5.0
+blobfile==3.0.0
+numpy==1.24.0 
+sentencepiece==0.2.0
+protobuf==6.30.2

vecalign/.gitignore

@@ -0,0 +1,10 @@
+build/
+dp_core.c*
+dp_core.html
+__pycache__/
+.idea
+*~
+.pytest_cache/
+venv/
+fairseq/
+scores/

vecalign/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 NYCU-RL-Bandits-Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vecalign/README.md

@@ -0,0 +1,102 @@
+# Plan2Align
+
+This is the official implementation for the paper **"Plan2Align: Predictive Planning Based Test-Time Preference Alignment in Paragraph-Level Machine Translation"**.
+
+## Environment Setup Guide for Plan2Align
+
+This document provides a step-by-step guide for setting up the environment required to run Plan2Align efficiently. Please follow the instructions below to ensure a smooth installation process.
+
+### 1. Create a Conda Virtual Environment (Recommended)
+
+It is highly recommended to use a Conda virtual environment to manage dependencies and avoid conflicts. Execute the following commands:
+
+```bash
+conda create --name plan2align python=3.9
+conda activate plan2align
+```
+
+### 2. Install VecAlign & SpaCy
+
+Plan2Align relies on VecAlign for alignment tasks. Please follow the installation instructions provided in the official repository:  
+[VecAlign GitHub Repository](https://github.com/thompsonb/vecalign)
+
+### 3. Configure Environment Variables for LASER
+
+LASER must be properly configured by setting up the required environment variables. Use the following steps:
+
+```bash
+nano ~/.bashrc
+export LASER="{PATH_TO_LASER}"
+source ~/.bashrc
+```
+
+Make sure to replace `{PATH_TO_LASER}` with the actual path where LASER is installed.
+
+### 4. Prepare API Key
+
+Plan2Align requires an API key for OpenAI services. Ensure that you have the necessary credentials set up:
+
+```python
+openai = OpenAI(
+    api_key='your-api-key',
+    base_url='your-base_url',
+)
+```
+
+Replace `'your-api-key'` and `'your-base_url'` with your actual API key and endpoint.
+
+### 5. Configure Reward Model
+
+Plan2Align utilizes a reward model for alignment tasks. Ensure that you modify the following paths in your reward model setup before use:
+
+```python
+self.RM = AutoModelForCausalLMWithValueHead.from_pretrained(
+    '../<path-to-rm>',
+    torch_dtype=torch.bfloat16
+).to(self.device)
+
+value_head_weights = load_file("../<path-to-value_head>")
+```
+
+Replace `<path-to-rm>` and `<path-to-value_head>` with the correct file paths in your system.
+
+Before running the program, you can use `set_translation_model("rm")` to make Plan2Align perform alignment based on the reward model.
+
+### 6. Running Plan2Align
+
+For ease of testing Plan2Align, we provide a small preference model for alignment. You can download its weights from the following link:  
+[Download Weights](https://drive.google.com/file/d/1us3pBmnJseI0-lozh999dDraql9m03im/view?usp=sharing).  
+Place it directly in the project directory, and use `set_translation_model("pm")` in `plan2align.py` to utilize it.
+
+Regarding datasets, we used the dataset from [Hugging Face](https://huggingface.co/datasets/huckiyang/zh-tw-en-us-nv-tech-blog-v1) and for validation. We selected longer, semantically structured samples from it, created a `valid_zh_en.csv`, and performed Chinese-to-English translation tasks.
+
+To validate that Plan2Align is correctly installed and configured, execute the following command:
+
+```bash
+python plan2align.py \
+    --input_file "valid_en_ja.csv" \
+    --rm "metricx" \
+    --src_language English \
+    --task_language Japanese \
+    --threshold 0.7 \
+    --max_iterations 6 \
+    --good_ref_contexts_num 5 \
+    --cuda_num 0
+```
+
+### 7. Evaluation Process
+
+---
+
+## Citation
+
+If you would like to cite this work, please use the following BibTeX entry:
+
+```bibtex
+@article{wang2025plan2align,
+  title={Plan2Align: Predictive Planning Based Test-Time Preference Alignment in Paragraph-Level Machine Translation},
+  author={Wang, Kuang-Da and Chen, Teng-Ruei and Hung, Yu Heng and Ding, Shuoyang and Wu, Yueh-Hua and Wang, Yu-Chiang Frank and Yang, Chao-Han Huck and Peng, Wen-Chih and Hsieh, Ping-Chun},
+  journal={arXiv preprint arXiv:2502.20795},
+  year={2025}
+}
+```

vecalign/dp_core.pyx

@@ -0,0 +1,411 @@
+# cython: language_level=3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import numpy as np
+
+cimport numpy as np
+cimport cython
+
+
+def make_x_y_offsets(alignment_types):
+    # alignment types for which we will precompute costs
+
+    # deletion/insertion is added later
+    for x, y in alignment_types:
+        assert (x > 0)
+        assert (y > 0)
+
+    x_offsets = np.array([x for x, y in alignment_types], dtype=np.int32)  # MUST **NOT** INCLUDE (0,1), (1,0)
+    y_offsets = np.array([y for x, y in alignment_types], dtype=np.int32)  # MUST **NOT** INCLUDE (0,1), (1,0)
+    return x_offsets, y_offsets
+
+
+def make_dense_costs(np.ndarray[float, ndim=3] vecs0,  # itput
+                     np.ndarray[float, ndim=3] vecs1,  # input
+                     np.ndarray[float, ndim=2] norm0,  # input
+                     np.ndarray[float, ndim=2] norm1,  # input
+                     int offset0 = 0,  # index into vecs0/norms0
+                     int offset1 = 0,  # index into vecs1/norms1
+                     ):
+    """
+    Make a full N*M feature matrix. By default, makes 1-1 alignments, 
+       can build others by specifying offset0, offset1 to index into
+       vecs0, norms0 and vecs1, norms1 respectivly. 
+    """
+    assert vecs0.shape[0] > offset0
+    assert vecs1.shape[0] > offset1
+    assert norm0.shape[0] > offset0
+    assert norm1.shape[0] > offset1
+
+    cdef int size0 = np.shape(vecs0)[1]
+    assert norm0.shape[1] == size0
+
+    cdef int size1 = np.shape(vecs1)[1]
+    assert norm1.shape[1] == size1
+
+    cdef int vecsize = np.shape(vecs0)[2]
+    assert vecs1.shape[2] == vecsize
+
+    cdef int xi, yi
+    cdef float sumx
+
+    cdef np.ndarray[float, ndim=2] costs = np.empty((size0, size1), dtype=np.float32)
+
+    for xi in range(size0):
+        for yi in range(size1):
+            sumx = 0.0
+            for jj in range(vecsize):
+                sumx += vecs0[offset0, xi, jj] * vecs1[offset1, yi, jj]
+
+            costs[xi, yi] = 2.0 * (1.0 - sumx) / (1e-6 + norm0[offset0, xi] + norm1[offset1, yi])
+            # normalize by alignment type  
+            costs[xi, yi] = costs[xi, yi] * (offset0 + 1) * (offset1 + 1)
+
+    return costs
+
+
+def dense_dp(np.ndarray[float, ndim=2] alignment_cost, float pen):
+    """
+    Compute cost matrix (csum) and backpointers (bp) 
+    from full 2-D 1-1 alignment costs matrix (alignment_cost) 
+    """
+
+    size0 = alignment_cost.shape[0]
+    size1 = alignment_cost.shape[1]
+    # csum and traceback matrix are both on nodes
+    #   so they are +1 in each dimension compared to the jump costs matrix
+    # For anything being used in accumulation, use float64
+    cdef np.ndarray[double, ndim=2] csum = np.empty((size0 + 1, size1 + 1), dtype=np.float64)
+    cdef np.ndarray[int, ndim=2] bp = np.empty((size0 + 1, size1 + 1), dtype=np.int32)
+
+    # bp and csum are nodes, 
+    #   while alignment_cost is the cost of going between the nodes
+    # Size of nodes should be one larger than alignment costs
+    b0, b1 = np.shape(bp)
+    c0, c1 = np.shape(csum)
+    j0, j1 = np.shape(alignment_cost)
+    assert (b0 == c0 == j0 + 1)
+    assert (b1 == c1 == j1 + 1)
+
+    cdef int cmax = np.shape(csum)[1]
+    cdef int rmax = np.shape(csum)[0]
+    cdef int c, r
+    cdef double cost0, cost1, cost2
+
+    # initialize the all c-direction deletion path
+    for c in range(cmax):
+        csum[0, c] = c * pen
+        bp[0, c] = 1
+
+    # initialize the all r-direction deletion path
+    for r in range(rmax):
+        csum[r, 0] = r * pen
+        bp[r, 0] = 2
+
+    # Initial cost is 0.0
+    csum[0, 0] = 0.0  # noop
+    bp[0, 0] = 4  # should not matter
+
+    # Calculate the rest recursively
+    for c in range(1, cmax):
+        for r in range(1, rmax):
+
+            # alignment_cost indexes are off by 1 wrt
+            #   csum/bp, since csum/bp are nodes
+            cost0 = csum[r - 1, c - 1] + alignment_cost[r - 1, c - 1]
+            cost1 = csum[r, c - 1] + pen
+            cost2 = csum[r - 1, c] + pen
+
+            csum[r, c] = cost0
+            bp[r, c] = 0
+
+            if cost1 < csum[r, c]:
+                csum[r, c] = cost1
+                bp[r, c] = 1
+            if cost2 < csum[r, c]:
+                csum[r, c] = cost2
+                bp[r, c] = 2
+
+    return csum, bp
+
+
+def score_path(np.ndarray[int, ndim=1] xx,
+               np.ndarray[int, ndim=1] yy,
+               np.ndarray[float, ndim=1] norm1,
+               np.ndarray[float, ndim=1] norm2,
+               np.ndarray[float, ndim=2] vecs1,
+               np.ndarray[float, ndim=2] vecs2,
+               np.ndarray[float, ndim=1] out):
+    cdef int xi, yi, ii, jj
+    cdef float outx
+    cdef int lenxy = xx.shape[0]
+    cdef int vecsize = vecs1.shape[1]
+
+    for ii in range(lenxy):
+        xi = xx[ii]
+        yi = yy[ii]
+        outx = 0.0
+        for jj in range(vecsize):
+            outx += vecs1[xi, jj] * vecs2[yi, jj]
+        out[ii] = 2.0 * (1.0 - outx) / (norm1[xi] + norm2[yi])
+
+
+# Bounds checking and wraparound slow things down by about 2x
+# Division by 0 checking has minimal speed impact
+@cython.boundscheck(False)  # turn off bounds-checking for entire function
+@cython.wraparound(False)  # turn off negative index wrapping for entire function
+@cython.cdivision(True)  # use c-style division (no division-by-zero check)
+def make_sparse_costs(np.ndarray[float, ndim=3] vecs0,  # intput: num aligns X num sents X dim
+                      np.ndarray[float, ndim=3] vecs1,  # input
+                      np.ndarray[float, ndim=2] norms0,  # intput: num aligns X num sents
+                      np.ndarray[float, ndim=2] norms1,  # input
+                      x_y_path,
+                      alignment_types,
+                      int width_over2):
+    """
+    Make features for DP, *for lines running across approximate path*, *for each alignment type*
+    x_offsets, y_offsets should not include (0,1), (1,0)
+
+    Basically, we take the feature matrix, rotate it 45 degress, 
+       and compute a "wavy" matrix for the features.
+    It's like the diagonal but it moves around to hopefully always include the true path.
+    """
+
+    cdef np.ndarray[int, ndim=2] x_y_path_ = np.array(x_y_path).astype(np.int32)
+
+    assert (vecs0.shape[0] == norms0.shape[0])
+    assert (vecs1.shape[0] == norms1.shape[0])
+
+    assert (vecs0.shape[1] == norms0.shape[1])
+    assert (vecs1.shape[1] == norms1.shape[1])
+
+    # check how many overlaps vectors were passed in
+    num_overlaps_in_vecs0 = vecs0.shape[0]
+    num_overlaps_in_vecs1 = vecs1.shape[0]
+
+    # check how many overlaps were requested
+    # edge case: alignment_types could be empty
+    #    In that case, we should just return insertions/deletions
+    #    and max_x_overlap == max_y_overlap == 0
+    max_x_overlap = max([0] + [x for x, y in alignment_types])  # add [0] in case alignment_types is empty
+    max_y_overlap = max([0] + [y for x, y in alignment_types])  # add [0] in case alignment_types is empty
+
+    # note: alignment types are specified 1-based, but vectors are stored 0-based
+    if max_x_overlap > num_overlaps_in_vecs0:
+        raise Exception('%d x overlaps requrested (via alignment_types), but vecs0 only has %d' % (
+            max_x_overlap, num_overlaps_in_vecs0))
+    if max_y_overlap > num_overlaps_in_vecs1:
+        raise Exception('%d y overlaps requrested (via alignment_types), but vecs1 only has %d' % (
+            max_y_overlap, num_overlaps_in_vecs1))
+
+    # number of sentences in each document
+    cdef int xsize = vecs0.shape[1]
+    cdef int ysize = vecs1.shape[1]
+
+    # vector diminsions should match
+    assert (vecs0.shape[2] == vecs1.shape[2])
+
+    cdef np.ndarray[int, ndim=1] x_offsets, y_offsets
+    x_offsets, y_offsets = make_x_y_offsets(alignment_types)
+
+    # reserve outputs
+    a_len = x_y_path_.shape[0]
+    b_len = 2 * width_over2
+    cdef np.ndarray[float, ndim=3] a_b_feats = np.empty((len(alignment_types), a_len, b_len), dtype=np.float32)
+    cdef np.ndarray[int, ndim=1] b_offset = np.empty(a_len).astype(np.int32)
+
+    cdef int x, y, aa, bb, xx, yy, a_idx, b_idx, bb2, x_offset, y_offset, ii_align, x_offset_idx, y_offset_idx
+    cdef int vecsize = vecs0.shape[2]
+    cdef int num_alignments = x_offsets.shape[0]
+
+    cdef float sumx, feat
+    cdef float inf = np.inf
+
+    for ii in range(x_y_path_.shape[0]):
+        x = x_y_path_[ii, 0]
+        y = x_y_path_[ii, 1]
+
+        # convert xy to ab cords
+        aa = x + y
+        bb = y
+
+        a_idx = aa
+        b_offset[aa] = bb - width_over2
+        for b_idx, bb2 in enumerate(range(bb - width_over2, bb + width_over2)):
+            # convert ab to xy cords
+            xx = aa - bb2
+            yy = bb2
+
+            for ii_align in range(num_alignments):
+                x_offset = x_offsets[ii_align]
+                x_offset_idx = x_offset - 1  # overlaps start at 1, vectors stored 0-based
+                y_offset = y_offsets[ii_align]
+                y_offset_idx = y_offset - 1
+
+                if 0 <= xx < xsize and 0 <= yy < ysize:
+                    sumx = 0.0
+                    for jj in range(vecsize):
+                        sumx += vecs0[x_offset_idx, xx, jj] * vecs1[y_offset_idx, yy, jj]
+                    feat = 2.0 * x_offset * y_offset * (1.0 - sumx) / (
+                            1e-6 + norms0[x_offset_idx, xx] + norms1[y_offset_idx, yy])
+
+                else:
+                    feat = inf
+
+                a_b_feats[ii_align, a_idx, b_idx] = feat
+
+    return a_b_feats, b_offset
+
+
+def sparse_dp(np.ndarray[float, ndim=3] a_b_costs,
+              np.ndarray[int, ndim=1] b_offset_in,
+              alignment_types,
+              double del_penalty,
+              int x_in_size,
+              int y_in_size):
+    """
+    Do DP along a path, using features saved off along path.
+    x_offsets, y_offsets should not include (0,1), (1,0)
+
+    xsize, ysize refer to the costs a_b_csum, but in x/y space
+
+    As in the simpler full-DP case, 
+       we compute cumulative costs and backpointers on notes,
+       and there are COSTS associated with moving between them.
+
+    This means the size of the notes  +1,+1 larger (in x,y) than the COSTS.
+    
+    So the size of a_b_csum, a_b_xp, a_b_yp are all one larger in x and y compared to the costs
+
+    In order to save memory (and time, vs a sparse matrix with hashes to look up values), let:
+             a = x + y
+             b = x - y
+
+    b_offsets tells us how far from the left edge the features are computed for.
+         basically it's like we are computing along the diagonal, 
+         but we shift the diagonal around based on our belief
+         about where the alignments are. 
+
+    b_offsets is used for both costs AND csum, backpointers, so it needs to be 
+        +2 longer (it is in the a-direction) than the costs (in the a direction)
+
+    """
+    cdef np.ndarray[int, ndim=1] x_offsets, y_offsets
+    x_offsets, y_offsets = make_x_y_offsets(alignment_types)
+
+    # make x/y offsets, including (0,1), (1,), i.e. including deletion and insertion
+    x_offsets = np.concatenate([x_offsets, np.array([0, 1], dtype=np.int32)])
+    y_offsets = np.concatenate([y_offsets, np.array([1, 0], dtype=np.int32)])
+
+    cdef int a_in_size = a_b_costs.shape[1]
+    cdef int b_in_size = a_b_costs.shape[2]
+
+    cdef int a_out_size = a_in_size + 2
+    cdef int b_out_size = b_in_size
+
+    cdef int x_out_size = x_in_size + 1
+    cdef int y_out_size = y_in_size + 1
+
+    # costs are the costs of going between nodes.
+    # in x,y for the nodes, we basically add a buffer 
+    #   at x=0 and y=0, and shift the cost by (x=+1,y=+1)
+    # In a,b space, this means adding two points (for the buffer)
+    #      at the beginning, and shifting by (a=+0,b=+1) since
+    #      a=x+y and b=y
+    # for the first two points, we can simply replicate the
+    #    original b_offset, since it should be -width_over2
+    # i.e. b_offset_in[0] == -width_over2
+    extra_two_points = np.array([b_offset_in[0], b_offset_in[0]], dtype=np.int32)
+    cdef np.ndarray[int, ndim=1] b_offset_out = np.concatenate([extra_two_points, b_offset_in + 1])
+
+    # outputs
+    # For anything being used in accumulation, use float64
+    cdef np.ndarray[double, ndim=2] a_b_csum = np.zeros((a_in_size + 2, b_in_size),
+                                                        dtype=np.float64) + np.inf  # error cumulative sum
+    cdef np.ndarray[int, ndim=2] a_b_xp = np.zeros((a_in_size + 2, b_in_size), dtype=np.int32) - 2  # backpointer for x
+    cdef np.ndarray[int, ndim=2] a_b_yp = np.zeros((a_in_size + 2, b_in_size), dtype=np.int32) - 2  # backpointer for y
+
+    cdef int num_alignments = x_offsets.shape[0]
+    cdef double inf = np.inf
+    cdef int xx_out, yy_out, ii_align, x_offset, y_offset
+    cdef int aa_in_cost, bb_in_cost, aa_out, bb_out, aa_out_prev, bb_out_prev, xx_in_cost, yy_in_cost, xx_out_prev, yy_out_prev
+
+    cdef double alignment_cost, total_cost, prev_cost
+
+    # increasing in a is the same as going along diagonals in x/y, so DP order works
+    #  (and any ordering is fine in b - nothing depends on values adjacent on diagonal in x/y)
+    for aa_out in range(a_in_size + 2):
+        for bb_out in range(b_in_size):
+            #xx_out, yy_out = ab2xy_w_offset(aa_out, bb_out, b_offset_out)
+            yy_out = bb_out + b_offset_out[aa_out]
+            xx_out = aa_out - yy_out
+
+            # edge case: all deletions in y-direction
+            if xx_out == 0 and 0 <= yy_out < y_out_size:
+                a_b_csum[aa_out, bb_out] = del_penalty * yy_out
+                a_b_xp[aa_out, bb_out] = 0
+                a_b_yp[aa_out, bb_out] = 1
+
+            # edge case: all deletions in x-direction
+            elif yy_out == 0 and 0 <= xx_out < x_out_size:
+                a_b_csum[aa_out, bb_out] = del_penalty * xx_out
+                a_b_xp[aa_out, bb_out] = 1
+                a_b_yp[aa_out, bb_out] = 0
+
+            else:
+                # initialize output to inf
+                a_b_csum[aa_out, bb_out] = inf
+                a_b_xp[aa_out, bb_out] = -42
+                a_b_yp[aa_out, bb_out] = -42
+
+                for ii_align in range(num_alignments):
+                    x_offset = x_offsets[ii_align]
+                    y_offset = y_offsets[ii_align]
+
+                    # coords of location of alignment cost, in input x/y space
+                    xx_in_cost = xx_out - 1  # features were front padded,
+                    yy_in_cost = yy_out - 1  #   so offset is always 1
+
+                    # the coords of location of previous cumsum cost, in input x/y space
+                    xx_out_prev = xx_out - x_offset
+                    yy_out_prev = yy_out - y_offset
+
+                    if 0 <= xx_in_cost < x_in_size and 0 <= yy_in_cost < y_in_size and 0 <= xx_out_prev < x_out_size and 0 <= yy_out_prev < y_out_size:
+                        # convert x,y to a,b
+                        aa_in_cost = xx_in_cost + yy_in_cost
+                        bb_in_cost = yy_in_cost - b_offset_in[aa_in_cost]
+
+                        aa_out_prev = xx_out_prev + yy_out_prev
+                        bb_out_prev = yy_out_prev - b_offset_out[aa_out_prev]
+
+                        if 0 <= aa_in_cost < a_in_size and 0 <= bb_in_cost < b_in_size and 0 <= aa_out_prev < a_out_size and 0 <= bb_out_prev < b_out_size:
+                            if x_offset == 0 or y_offset == 0:
+                                alignment_cost = del_penalty
+                            else:
+                                alignment_cost = a_b_costs[ii_align, aa_in_cost, bb_in_cost]
+
+                            prev_cost = a_b_csum[aa_out_prev, bb_out_prev]
+
+                            total_cost = prev_cost + alignment_cost
+
+                            if total_cost < a_b_csum[aa_out, bb_out]:
+                                a_b_csum[aa_out, bb_out] = total_cost
+                                a_b_xp[aa_out, bb_out] = x_offset
+                                a_b_yp[aa_out, bb_out] = y_offset
+
+    return a_b_csum, a_b_xp, a_b_yp, b_offset_out

vecalign/dp_utils.py

@@ -0,0 +1,679 @@
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import logging
+import sys
+from ast import literal_eval
+from collections import OrderedDict
+from math import ceil
+from time import time
+
+import numpy as np
+
+import pyximport
+pyximport.install(setup_args={'include_dirs':np.get_include()}, inplace=True, reload_support=True)
+
+from dp_core import make_dense_costs, score_path, sparse_dp, make_sparse_costs, dense_dp
+
+logger = logging.getLogger('vecalign')  # set up in vecalign.py
+
+
+def preprocess_line(line):
+    line = line.strip()
+    if len(line) == 0:
+        line = 'BLANK_LINE'
+    return line
+
+
+def yield_overlaps(lines, num_overlaps):
+    lines = [preprocess_line(line) for line in lines]
+    for overlap in range(1, num_overlaps + 1):
+        for out_line in layer(lines, overlap):
+            # check must be here so all outputs are unique
+            out_line2 = out_line[:10000]  # limit line so dont encode arbitrarily long sentences
+            yield out_line2
+
+
+def read_in_embeddings(text_file, embed_file):
+    """
+    Given a text file with candidate sentences and a corresponing embedding file,
+       make a maping from candidate sentence to embedding index, 
+       and a numpy array of the embeddings
+    """
+    sent2line = dict()
+    with open(text_file, 'rt', encoding="utf-8") as fin:
+        for ii, line in enumerate(fin):
+            if line.strip() in sent2line:
+                raise Exception('got multiple embeddings for the same line')
+            sent2line[line.strip()] = ii
+
+    line_embeddings = np.fromfile(embed_file, dtype=np.float32, count=-1)
+    if line_embeddings.size == 0:
+        raise Exception('Got empty embedding file')
+
+    laser_embedding_size = line_embeddings.size // len(sent2line)  # currently hardcoded to 1024
+    if laser_embedding_size != 1024:
+        logger.warning('expected an embedding size of 1024, got %s', laser_embedding_size)
+    logger.info('laser_embedding_size determined to be %d', laser_embedding_size)
+    line_embeddings.resize(line_embeddings.shape[0] // laser_embedding_size, laser_embedding_size)
+    return sent2line, line_embeddings
+
+
+def make_doc_embedding(sent2line, line_embeddings, lines, num_overlaps):
+    """
+    lines: sentences in input document to embed
+    sent2line, line_embeddings: precomputed embeddings for lines (and overlaps of lines)
+    """
+
+    lines = [preprocess_line(line) for line in lines]
+
+    vecsize = line_embeddings.shape[1]
+
+    vecs0 = np.empty((num_overlaps, len(lines), vecsize), dtype=np.float32)
+
+    for ii, overlap in enumerate(range(1, num_overlaps + 1)):
+        for jj, out_line in enumerate(layer(lines, overlap)):
+            try:
+                line_id = sent2line[out_line]
+            except KeyError:
+                logger.warning('Failed to find overlap=%d line "%s". Will use random vector.', overlap, out_line)
+                line_id = None
+
+            if line_id is not None:
+                vec = line_embeddings[line_id]
+            else:
+                vec = np.random.random(vecsize) - 0.5
+                vec = vec / np.linalg.norm(vec)
+
+            vecs0[ii, jj, :] = vec
+
+    return vecs0
+
+
+def make_norm1(vecs0):
+    """
+    make vectors norm==1 so that cosine distance can be computed via dot product
+    """
+    for ii in range(vecs0.shape[0]):
+        for jj in range(vecs0.shape[1]):
+            norm = np.sqrt(np.square(vecs0[ii, jj, :]).sum())
+            vecs0[ii, jj, :] = vecs0[ii, jj, :] / (norm + 1e-5)
+
+
+def layer(lines, num_overlaps, comb=' '):
+    """
+    make front-padded overlapping sentences
+    """
+    if num_overlaps < 1:
+        raise Exception('num_overlaps must be >= 1')
+    out = ['PAD', ] * min(num_overlaps - 1, len(lines))
+    for ii in range(len(lines) - num_overlaps + 1):
+        out.append(comb.join(lines[ii:ii + num_overlaps]))
+    return out
+
+
+def read_alignments(fin):
+    alignments = []
+    with open(fin, 'rt', encoding="utf-8") as infile:
+        for line in infile:
+            fields = [x.strip() for x in line.split(':') if len(x.strip())]
+            if len(fields) < 2:
+                raise Exception('Got line "%s", which does not have at least two ":" separated fields' % line.strip())
+            try:
+                src = literal_eval(fields[0])
+                tgt = literal_eval(fields[1])
+            except:
+                raise Exception('Failed to parse line "%s"' % line.strip())
+            alignments.append((src, tgt))
+
+    # I know bluealign files have a few entries entries missing,
+    #   but I don't fix them in order to be consistent previous reported scores
+    return alignments
+
+
+def print_alignments(alignments, scores=None, src_lines=None, tgt_lines=None, ofile=sys.stdout):
+    if scores is None:
+        scores = [None for _ in alignments]
+    for (x, y), s in zip(alignments, scores):
+        if s is None:
+            print('%s:%s' % (x, y), file=ofile)
+        else:
+            print('%s:%s:%.6f' % (x, y, s), file=ofile)
+        if src_lines is not None and tgt_lines is not None:
+            print(' '*40, 'SRC: ', ' '.join([src_lines[i].replace('\n', ' ').strip() for i in x]), file=ofile)
+            print(' '*40, 'TGT: ', ' '.join([tgt_lines[i].replace('\n', ' ').strip() for i in y]), file=ofile)
+
+
+class DeletionKnob(object):
+    """
+    A good deletion penalty is dependent on normalization, and probably language, domain, etc, etc
+    I want a way to control deletion penalty that generalizes well...
+    Sampling costs and use percentile seems to work fairly well.
+    """
+    def __init__(self, samp, res_min, res_max):
+
+        self.res_min = res_min
+        self.res_max = res_max
+
+        if self.res_min >= self.res_max:
+            logger.warning('res_max <= res_min, increasing it')
+            self.res_max = self.res_min + 1e-4
+
+        num_bins = 1000
+        num_pts = 30
+
+        self.hist, self.bin_edges = np.histogram(samp, bins=num_bins,
+                                                 range=[self.res_min, self.res_max],
+                                                 density=True)
+
+        dx = self.bin_edges[1] - self.bin_edges[0]
+        self.cdf = np.cumsum(self.hist) * dx
+
+        interp_points = [(0, self.res_min), ]
+        for knob_val in np.linspace(0, 1, num_pts - 1)[1:-1]:
+            cdf_idx = np.searchsorted(self.cdf, knob_val)
+            cdf_val = self.res_min + cdf_idx / float(num_bins) * (self.res_max - self.res_min)
+            interp_points.append((knob_val, cdf_val))
+        interp_points.append((1, self.res_max))
+        self.x, self.y = zip(*interp_points)
+
+    def percentile_frac_to_del_penalty(self, knob_val):
+        del_pen = np.interp([knob_val], self.x, self.y)[0]
+        return del_pen
+
+
+def make_alignment_types(max_alignment_size):
+    # return list of all (n,m) where n+m <= max_alignment_size
+    # does not include deletions, i.e. (1, 0) or (0, 1)
+    alignment_types = []
+    for x in range(1, max_alignment_size):
+        for y in range(1, max_alignment_size):
+            if x + y <= max_alignment_size:
+                alignment_types.append((x, y))
+    return alignment_types
+
+
+def make_one_to_many_alignment_types(max_alignment_size):
+    # return list of all (1, m) where m <= max_alignment_size
+    # does not include deletions, i.e. (1, 0) or (0, 1)
+    alignment_types = []
+    for m in range(1, max_alignment_size + 1):
+        alignment_types.append((1, m))
+    return alignment_types
+
+
+def ab2xy_w_offset(aa, bb_idx, bb_offset):
+    bb_from_side = bb_idx + bb_offset[aa]
+    xx = aa - bb_from_side
+    yy = bb_from_side
+    return (xx, yy)
+
+
+def xy2ab_w_offset(xx, yy, bb_offset):
+    aa = xx + yy
+    bb_from_side = yy
+    bb = bb_from_side - bb_offset[aa]
+    return aa, bb
+
+
+def process_scores(scores, alignments):
+    # floating point sometimes gives negative numbers, which is a little unnerving ...
+    scores = np.clip(scores, a_min=0, a_max=None)
+
+    for ii, (x_algn, y_algn) in enumerate(alignments):
+        # deletion penalty is pretty arbitrary, just report 0
+        if len(x_algn) == 0 or len(y_algn) == 0:
+            scores[ii] = 0.0
+        # report sores un-normalized by alignment sizes
+        #    (still normalized with random vectors, though)
+        else:
+            scores[ii] = scores[ii] / len(x_algn) / len(y_algn)
+
+    return scores
+
+
+def sparse_traceback(a_b_csum, a_b_xp, a_b_yp, b_offset, xsize, ysize):
+    alignments = []
+    xx = xsize
+    yy = ysize
+
+    cum_costs = []
+
+    while True:
+        aa, bb = xy2ab_w_offset(xx, yy, b_offset)
+
+        cum_costs.append(a_b_csum[aa, bb])
+
+        xp = a_b_xp[aa, bb]
+        yp = a_b_yp[aa, bb]
+
+        if xx == yy == 0:
+            break
+
+        if xx < 0 or yy < 0:
+            raise Exception('traceback bug')
+
+        x_side = list(range(xx - xp, xx))
+        y_side = list(range(yy - yp, yy))
+        alignments.append((x_side, y_side))
+
+        xx = xx - xp
+        yy = yy - yp
+
+    alignments.reverse()
+    cum_costs.reverse()
+    costs = np.array(cum_costs[1:]) - np.array(cum_costs[:-1])
+    # "costs" are scaled by x_alignment_size * y_alignment_size
+    #     and the cost of a deletion is del_penalty
+    # "scores": 0 for deletion/insertion, 
+    #    and cosine distance, *not* scaled 
+    #    by len(x_alignment)*len(y_alignment)
+    scores = process_scores(scores=costs, alignments=alignments)
+
+    return alignments, scores
+
+
+def dense_traceback(x_y_tb):
+    xsize, ysize = x_y_tb.shape
+
+    xx = xsize - 1
+    yy = ysize - 1
+
+    alignments = []
+    while True:
+        if xx == yy == 0:
+            break
+        bp = x_y_tb[xx, yy]
+        if bp == 0:
+            xp, yp = 1, 1
+            alignments.append(([xx - 1], [yy - 1]))
+        elif bp == 1:
+            xp, yp = 0, 1
+            alignments.append(([], [yy - 1]))
+        elif bp == 2:
+            xp, yp = 1, 0
+            alignments.append(([xx - 1], []))
+        else:
+            raise Exception('got unknown value')
+
+        xx = xx - xp
+        yy = yy - yp
+
+    alignments.reverse()
+
+    return alignments
+
+
+def append_slant(path, xwidth, ywidth):
+    """
+    Append quantized approximation to a straight line
+       from current x,y to a point at (x+xwidth, y+ywidth)
+    """
+    NN = xwidth + ywidth
+    xstart, ystart = path[-1]
+    for ii in range(1, NN + 1):
+        x = xstart + round(xwidth * ii / NN)
+        y = ystart + round(ywidth * ii / NN)
+        # In the case of ties we want them to round differently,
+        #   so explicitly make sure we take a step of 1, not 0 or 2
+        lastx, lasty = path[-1]
+        delta = x + y - lastx - lasty
+        if delta == 1:
+            path.append((x, y))
+        elif delta == 2:
+            path.append((x - 1, y))
+        elif delta == 0:
+            path.append((x + 1, y))
+
+
+def alignment_to_search_path(algn):
+    """
+    Given an alignment, make searchpath.
+    Searchpath must step exactly one position in x XOR y at each time step.
+    
+    In the case of a block of deletions, the order found by DP is not meaningful.
+    To make things consistent and to improve the probability of recovering 
+       from search errors, we search an approximately straight line
+       through a block of deletions. We do the same through a many-many 
+       alignment, even though we currently don't refine a many-many alignment...
+    """
+    path = [(0, 0), ]
+    xdel, ydel = 0, 0
+    ydel = 0
+    for x, y in algn:
+        if len(x) and len(y):
+            append_slant(path, xdel, ydel)
+            xdel, ydel = 0, 0
+            append_slant(path, len(x), len(y))
+        elif len(x):
+            xdel += len(x)
+        elif len(y):
+            ydel += len(y)
+
+    append_slant(path, xdel, ydel)
+
+    return path
+
+
+def extend_alignments(course_alignments, size0, size1):
+    """
+    extend alignments to include new endpoints size0, size1
+    if alignments are larger than size0/size1, raise exception
+    """
+    # could be a string of deletions or insertions at end, so cannot just grab last one
+    xmax = 0  # maximum x value in course_alignments
+    ymax = 0  # maximum y value in course_alignments
+    for x, y in course_alignments:
+        for xval in x:
+            xmax = max(xmax, xval)
+        for yval in y:
+            ymax = max(ymax, yval)
+
+    if xmax > size0 or ymax > size1:
+        raise Exception('asked to extend alignments but already bigger than requested')
+
+    # do not duplicate xmax/ymax, do include size0/size1 
+    extra_x = list(range(xmax + 1, size0 + 1))
+    extra_y = list(range(ymax + 1, size1 + 1))
+
+    logger.debug('extending alignments in x by %d and y by %d', len(extra_x), len(extra_y))
+
+    if len(extra_x) == 0:
+        for yval in extra_y:
+            course_alignments.append(([], [yval]))
+    elif len(extra_y) == 0:
+        for xval in extra_x:
+            course_alignments.append(([xval], []))
+    else:
+        course_alignments.append((extra_x, extra_y))
+
+
+def upsample_alignment(algn):
+    def upsample_one_alignment(xx):
+        return list(range(min(xx) * 2, (max(xx) + 1) * 2))
+
+    new_algn = []
+    for xx, yy in algn:
+        if len(xx) == 0:
+            for yyy in upsample_one_alignment(yy):
+                new_algn.append(([], [yyy]))
+        elif len(yy) == 0:
+            for xxx in upsample_one_alignment(xx):
+                new_algn.append(([xxx], []))
+        else:
+            new_algn.append((upsample_one_alignment(xx), upsample_one_alignment(yy)))
+    return new_algn
+
+
+def make_del_knob(e_laser,
+                  f_laser,
+                  e_laser_norms,
+                  f_laser_norms,
+                  sample_size):
+    e_size = e_laser.shape[0]
+    f_size = f_laser.shape[0]
+
+    if e_size > 0 and f_size > 0 and sample_size > 0:
+
+        if e_size * f_size < sample_size:
+            # dont sample, just compute full matrix
+            sample_size = e_size * f_size
+            x_idxs = np.zeros(sample_size, dtype=np.int32)
+            y_idxs = np.zeros(sample_size, dtype=np.int32)
+            c = 0
+            for ii in range(e_size):
+                for jj in range(f_size):
+                    x_idxs[c] = ii
+                    y_idxs[c] = jj
+                    c += 1
+        else:
+            # get random samples
+            x_idxs = np.random.choice(range(e_size), size=sample_size, replace=True).astype(np.int32)
+            y_idxs = np.random.choice(range(f_size), size=sample_size, replace=True).astype(np.int32)
+
+        # output
+        random_scores = np.empty(sample_size, dtype=np.float32)
+
+        score_path(x_idxs, y_idxs,
+                   e_laser_norms, f_laser_norms,
+                   e_laser, f_laser,
+                   random_scores, )
+
+        min_score = 0
+        max_score = max(random_scores)  # could bump this up... but its probably fine
+
+    else:
+        # Not much we can do here...
+        random_scores = np.array([0.0, 0.5, 1.0])  # ???
+        min_score = 0
+        max_score = 1  # ????
+
+    del_knob = DeletionKnob(random_scores, min_score, max_score)
+
+    return del_knob
+
+
+def compute_norms(vecs0, vecs1, num_samples, overlaps_to_use=None):
+    # overlaps_to_use = 10  # 10 matches before
+
+    overlaps1, size1, dim = vecs1.shape
+    overlaps0, size0, dim0 = vecs0.shape
+    assert (dim == dim0)
+
+    if overlaps_to_use is not None:
+        if overlaps_to_use > overlaps1:
+            raise Exception('Cannot use more overlaps than provided. You may want to re-run make_verlaps.py with a larger -n value')
+    else:
+        overlaps_to_use = overlaps1
+
+    samps_per_overlap = ceil(num_samples / overlaps_to_use)
+
+    if size1 and samps_per_overlap:
+        # sample other size (from all overlaps) to compre to this side
+        vecs1_rand_sample = np.empty((samps_per_overlap * overlaps_to_use, dim), dtype=np.float32)
+        for overlap_ii in range(overlaps_to_use):
+            idxs = np.random.choice(range(size1), size=samps_per_overlap, replace=True)
+            random_vecs = vecs1[overlap_ii, idxs, :]
+            vecs1_rand_sample[overlap_ii * samps_per_overlap:(overlap_ii + 1) * samps_per_overlap, :] = random_vecs
+
+        norms0 = np.empty((overlaps0, size0), dtype=np.float32)
+        for overlap_ii in range(overlaps0):
+            e_laser = vecs0[overlap_ii, :, :]
+            sim = np.matmul(e_laser, vecs1_rand_sample.T)
+            norms0[overlap_ii, :] = 1.0 - sim.mean(axis=1)
+
+    else:  # no samples, no normalization
+        norms0 = np.ones((overlaps0, size0)).astype(np.float32)
+
+    return norms0
+
+
+def downsample_vectors(vecs1):
+    a, b, c = vecs1.shape
+    half = np.empty((a, b // 2, c), dtype=np.float32)
+    for ii in range(a):
+        # average consecutive vectors
+        for jj in range(0, b - b % 2, 2):
+            v1 = vecs1[ii, jj, :]
+            v2 = vecs1[ii, jj + 1, :]
+            half[ii, jj // 2, :] = v1 + v2
+        # compute mean for all vectors
+        mean = np.mean(half[ii, :, :], axis=0)
+        for jj in range(0, b - b % 2, 2):
+            # remove mean
+            half[ii, jj // 2, :] = half[ii, jj // 2, :] - mean
+    # make vectors norm==1 so dot product is cosine distance
+    make_norm1(half)
+    return half
+
+
+def vecalign(vecs0,
+             vecs1,
+             final_alignment_types,
+             del_percentile_frac,
+             width_over2,
+             max_size_full_dp,
+             costs_sample_size,
+             num_samps_for_norm,
+             norms0=None,
+             norms1=None):
+    if width_over2 < 3:
+        logger.warning('width_over2 was set to %d, which does not make sense. increasing to 3.', width_over2)
+        width_over2 = 3
+
+    # make sure input embeddings are norm==1
+    make_norm1(vecs0)
+    make_norm1(vecs1)
+
+    # save off runtime stats for summary
+    runtimes = OrderedDict()
+
+    # Determine stack depth
+    s0, s1 = vecs0.shape[1], vecs1.shape[1]
+    max_depth = 0
+    while s0 * s1 > max_size_full_dp ** 2:
+        max_depth += 1
+        s0 = s0 // 2
+        s1 = s1 // 2
+
+    # init recursion stack
+    # depth is 0-based (full size is 0, 1 is half, 2 is quarter, etc)
+    stack = {0: {'v0': vecs0, 'v1': vecs1}}
+
+    # downsample sentence vectors
+    t0 = time()
+    for depth in range(1, max_depth + 1):
+        stack[depth] = {'v0': downsample_vectors(stack[depth - 1]['v0']),
+                        'v1': downsample_vectors(stack[depth - 1]['v1'])}
+    runtimes['Downsample embeddings'] = time() - t0
+
+    # compute norms for all depths, add sizes, add alignment types
+    t0 = time()
+    for depth in stack:
+        stack[depth]['size0'] = stack[depth]['v0'].shape[1]
+        stack[depth]['size1'] = stack[depth]['v1'].shape[1]
+        stack[depth]['alignment_types'] = final_alignment_types if depth == 0 else [(1, 1)]
+
+        if depth == 0 and norms0 is not None:
+            if norms0.shape != vecs0.shape[:2]:
+                print('norms0.shape:', norms0.shape)
+                print('vecs0.shape[:2]:', vecs0.shape[:2])
+                raise Exception('norms0 wrong shape')
+            stack[depth]['n0'] = norms0
+        else:
+            stack[depth]['n0'] = compute_norms(stack[depth]['v0'], stack[depth]['v1'], num_samps_for_norm)
+
+        if depth == 0 and norms1 is not None:
+            if norms1.shape != vecs1.shape[:2]:
+                print('norms1.shape:', norms1.shape)
+                print('vecs1.shape[:2]:', vecs1.shape[:2])
+                raise Exception('norms1 wrong shape')
+            stack[depth]['n1'] = norms1
+        else:
+            stack[depth]['n1'] = compute_norms(stack[depth]['v1'], stack[depth]['v0'], num_samps_for_norm)
+
+    runtimes['Normalize embeddings'] = time() - t0
+
+    # Compute deletion penalty for all depths
+    t0 = time()
+    for depth in stack:
+        stack[depth]['del_knob'] = make_del_knob(e_laser=stack[depth]['v0'][0, :, :],
+                                                 f_laser=stack[depth]['v1'][0, :, :],
+                                                 e_laser_norms=stack[depth]['n0'][0, :],
+                                                 f_laser_norms=stack[depth]['n1'][0, :],
+                                                 sample_size=costs_sample_size)
+        stack[depth]['del_penalty'] = stack[depth]['del_knob'].percentile_frac_to_del_penalty(del_percentile_frac)
+        logger.debug('del_penalty at depth %d: %f', depth, stack[depth]['del_penalty'])
+    runtimes['Compute deletion penalties'] = time() - t0
+    tt = time() - t0
+    logger.debug('%d x %d full DP make features: %.6fs (%.3e per dot product)',
+                 stack[max_depth]['size0'], stack[max_depth]['size1'], tt,
+                 tt / (stack[max_depth]['size0'] + 1e-6) / (stack[max_depth]['size1'] + 1e-6))
+    # full DP at maximum recursion depth
+    t0 = time()
+    stack[max_depth]['costs_1to1'] = make_dense_costs(stack[max_depth]['v0'],
+                                                      stack[max_depth]['v1'],
+                                                      stack[max_depth]['n0'],
+                                                      stack[max_depth]['n1'])
+
+    runtimes['Full DP make features'] = time() - t0
+    t0 = time()
+    _, stack[max_depth]['x_y_tb'] = dense_dp(stack[max_depth]['costs_1to1'], stack[max_depth]['del_penalty'])
+    stack[max_depth]['alignments'] = dense_traceback(stack[max_depth]['x_y_tb'])
+    runtimes['Full DP'] = time() - t0
+
+    # upsample the path up to the top resolution
+    compute_costs_times = []
+    dp_times = []
+    upsample_depths = [0, ] if max_depth == 0 else list(reversed(range(0, max_depth)))
+    for depth in upsample_depths:
+        if max_depth > 0:  # upsample previoius alignment to current resolution
+            course_alignments = upsample_alignment(stack[depth + 1]['alignments'])
+            # features may have been truncated when downsampleing, so alignment may need extended
+            extend_alignments(course_alignments, stack[depth]['size0'], stack[depth]['size1'])  # in-place
+        else:  # We did a full size 1-1 search, so search same size with more alignment types
+            course_alignments = stack[0]['alignments']
+
+        # convert couse alignments to a searchpath
+        stack[depth]['searchpath'] = alignment_to_search_path(course_alignments)
+
+        # compute ccosts for sparse DP
+        t0 = time()
+        stack[depth]['a_b_costs'], stack[depth]['b_offset'] = make_sparse_costs(stack[depth]['v0'], stack[depth]['v1'],
+                                                                                stack[depth]['n0'], stack[depth]['n1'],
+                                                                                stack[depth]['searchpath'],
+                                                                                stack[depth]['alignment_types'],
+                                                                                width_over2)
+
+        tt = time() - t0
+        num_dot_products = len(stack[depth]['b_offset']) * len(stack[depth]['alignment_types']) * width_over2 * 2
+        logger.debug('%d x %d sparse DP (%d alignment types, %d window) make features: %.6fs (%.3e per dot product)',
+                     stack[max_depth]['size0'], stack[max_depth]['size1'],
+                     len(stack[depth]['alignment_types']), width_over2 * 2,
+                     tt, tt / (num_dot_products + 1e6))
+
+        compute_costs_times.append(time() - t0)
+        t0 = time()
+        # perform sparse DP
+        stack[depth]['a_b_csum'], stack[depth]['a_b_xp'], stack[depth]['a_b_yp'], \
+        stack[depth]['new_b_offset'] = sparse_dp(stack[depth]['a_b_costs'], stack[depth]['b_offset'],
+                                                 stack[depth]['alignment_types'], stack[depth]['del_penalty'],
+                                                 stack[depth]['size0'], stack[depth]['size1'])
+
+        # performace traceback to get alignments and alignment scores
+        # for debugging, avoid overwriting stack[depth]['alignments']
+        akey = 'final_alignments' if depth == 0 else 'alignments'
+        stack[depth][akey], stack[depth]['alignment_scores'] = sparse_traceback(stack[depth]['a_b_csum'],
+                                                                                stack[depth]['a_b_xp'],
+                                                                                stack[depth]['a_b_yp'],
+                                                                                stack[depth]['new_b_offset'],
+                                                                                stack[depth]['size0'],
+                                                                                stack[depth]['size1'])
+        dp_times.append(time() - t0)
+
+    runtimes['Upsample DP compute costs'] = sum(compute_costs_times[:-1])
+    runtimes['Upsample DP'] = sum(dp_times[:-1])
+
+    runtimes['Final DP compute costs'] = compute_costs_times[-1]
+    runtimes['Final DP'] = dp_times[-1]
+
+    # log time stats
+    max_key_str_len = max([len(key) for key in runtimes])
+    for key in runtimes:
+        if runtimes[key] > 5e-5:
+            logger.info(key + ' took ' + '.' * (max_key_str_len + 5 - len(key)) + ('%.4fs' % runtimes[key]).rjust(7))
+
+    return stack

vecalign/long_context_eval.py

@@ -0,0 +1,1080 @@
+#!/usr/bin/env python3
+"""
+Long Context Evaluation
+
+This script performs the following steps:
+  1. Reads a specified CSV file containing the evaluation data.
+  2. Segments source, reference, and MT texts into sentences and sliding windows.
+  3. Generates overlaps and embeddings for alignment.
+  4. Runs vector alignment exploration and computes COMET and COMET-QE scores.
+  5. Aggregates the scores and saves the results.
+"""
+
+import os
+import re
+import json
+import csv
+import spacy
+import torch
+import random
+import argparse
+import numpy as np
+import pandas as pd
+import tempfile
+import subprocess
+import unicodedata
+from multiprocessing import Pool
+import datetime
+from typing import Optional
+
+# -----------------------------------------------------------------------------
+# Utility Functions
+# -----------------------------------------------------------------------------
+def set_seed(seed: int = 42) -> None:
+    """
+    Set the global random seed for reproducibility.
+
+    Args:
+        seed (int): Random seed (default is 42).
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+def normalize_text(text: str) -> str:
+    """
+    Normalize text using Unicode normalization (NFKC) to convert full-width characters to half-width.
+    Additional normalization (e.g., lowercasing) can be added if needed.
+
+    Args:
+        text (str): Input text.
+
+    Returns:
+        str: Normalized text.
+    """
+    normalized = unicodedata.normalize("NFKC", text)
+    # Uncomment the following if lowercase conversion is desired:
+    # normalized = normalized.lower()
+    return normalized
+
+
+def segment_sentences_by_punctuation(text: str, lang: str) -> list:
+    """
+    Segment text into sentences based on punctuation and add an end-of-sentence separator.
+
+    Args:
+        text (str): Input text (may contain multiple paragraphs).
+        lang (str): Language code (e.g., "zh", "en", "ru", "de").
+
+    Returns:
+        list: List of segmented sentences with the SEPARATOR appended.
+    """
+    segmented_sentences = []
+    paragraphs = text.split('\n')
+    for paragraph in paragraphs:
+        if paragraph.strip():
+            if lang == SRC_LANG:
+                doc = src_nlp(paragraph)
+            else:
+                doc = mt_nlp(paragraph)
+            for sent in doc.sents:
+                segmented_sentences.append(normalize_text(sent.text.strip()) + SEPARATOR)
+    return segmented_sentences
+
+
+def preprocess_sentences(sentences: list) -> str:
+    """
+    Preprocess sentences by removing the end-of-sentence token and joining them with newline characters.
+
+    Args:
+        sentences (list): List of sentences.
+
+    Returns:
+        str: Preprocessed text.
+    """
+    processed = [sentence.replace(SEPARATOR, "").strip() for sentence in sentences]
+    return "\n".join(processed)
+
+
+def generate_overlap_and_embedding(text: str) -> tuple:
+    """
+    Generate overlap and embedding data from text using temporary files.
+
+    Args:
+        text (str): Input text.
+
+    Returns:
+        tuple: (overlap_content (str), embeddings_content (bytes))
+    """
+    with tempfile.NamedTemporaryFile(delete=True, mode="w+", encoding="utf-8", suffix=".txt") as txt_file:
+        txt_file.write(text)
+        txt_file.flush()
+        txt_filename = txt_file.name
+        overlaps_file = txt_filename + ".overlaps"
+        embed_file = txt_filename + ".emb"
+
+        # Generate overlap data
+        subprocess.run(["./overlap.py", "-i", txt_filename, "-o", overlaps_file, "-n", "10"], check=True)
+        # Generate embedding data
+        subprocess.run(" ".join(["$LASER/tasks/embed/embed.sh", overlaps_file, embed_file]),
+                       shell=True, check=True)
+
+        with open(embed_file, "rb") as f:
+            embeddings_content = f.read()
+        with open(overlaps_file, "r", encoding="utf-8") as f:
+            overlap_content = f.read()
+
+    for need_to_del_file in [overlaps_file, embed_file]:
+        try:
+            os.remove(need_to_del_file)
+            print(f"Removed file: {need_to_del_file}")
+        except Exception as e:
+            print(f"Error removing {need_to_del_file}: {e}")
+
+    return overlap_content, embeddings_content
+
+
+def compute_alignment_stats(alignment_results: list) -> tuple:
+    """
+    Compute the average alignment cost (ignoring zero-cost alignments) and the zero-cost ratio.
+
+    Args:
+        alignment_results (list): List of alignment result strings in the format "[src]:[tgt]:cost".
+
+    Returns:
+        tuple: (average_cost (float), zero_cost_ratio (float))
+    """
+    costs = []
+    zero_cost_count = 0
+
+    for entry in alignment_results:
+        try:
+            cost = float(entry.split(":")[-1])
+            if cost == 0.0:
+                zero_cost_count += 1
+            else:
+                costs.append(cost)
+        except ValueError:
+            continue
+
+    avg_cost = sum(costs) / len(costs) if costs else 0.0
+    zero_cost_ratio = zero_cost_count / len(alignment_results) if alignment_results else 0.0
+
+    return avg_cost, zero_cost_ratio
+
+
+def run_vecalign_explore(src_text: str, tgt_text: str, src_overlap: str, tgt_overlap: str,
+                         src_embed: bytes, tgt_embed: bytes) -> list:
+    """
+    Explore the best vector alignment parameters and return the best alignments.
+
+    Args:
+        src_text (str): Source text.
+        tgt_text (str): Target text.
+        src_overlap (str): Overlap data for the source.
+        tgt_overlap (str): Overlap data for the target.
+        src_embed (bytes): Embedding data for the source.
+        tgt_embed (bytes): Embedding data for the target.
+
+    Returns:
+        list: Parsed best alignments as a list of tuples [(src_indices, tgt_indices), ...].
+    """
+    del_percentile_frac = 0.2
+    step_size = 0.005
+    prev_zero_cost_ratio = None
+    prev_avg_cost = None
+
+    best_avg_cost = float('inf')
+    best_del_percentile_frac = del_percentile_frac
+    best_zero_cost_ratio = 0.0
+    best_alignments = []
+
+    first_flag = True
+
+    with tempfile.NamedTemporaryFile(delete=True, mode="w+", encoding="utf-8", suffix=".txt") as src_file, \
+         tempfile.NamedTemporaryFile(delete=True, mode="w+", encoding="utf-8", suffix=".txt") as tgt_file, \
+         tempfile.NamedTemporaryFile(delete=True, mode="w+", encoding="utf-8", suffix=".overlaps") as src_overlap_file, \
+         tempfile.NamedTemporaryFile(delete=True, mode="w+", encoding="utf-8", suffix=".overlaps") as tgt_overlap_file, \
+         tempfile.NamedTemporaryFile(delete=True, mode="wb", suffix=".emb") as src_embed_file, \
+         tempfile.NamedTemporaryFile(delete=True, mode="wb", suffix=".emb") as tgt_embed_file:
+
+        src_file.write(src_text)
+        src_file.flush()
+        tgt_file.write(tgt_text)
+        tgt_file.flush()
+
+        src_overlap_file.write(src_overlap)
+        src_overlap_file.flush()
+        tgt_overlap_file.write(tgt_overlap)
+        tgt_overlap_file.flush()
+
+        src_embed_file.write(src_embed)
+        src_embed_file.flush()
+        tgt_embed_file.write(tgt_embed)
+        tgt_embed_file.flush()
+
+        while del_percentile_frac > 0:
+            result = subprocess.run(
+                [
+                    "./vecalign.py",
+                    "--alignment_max_size", "8",
+                    "--del_percentile_frac", str(del_percentile_frac),
+                    "--src", src_file.name,
+                    "--tgt", tgt_file.name,
+                    "--src_embed", src_overlap_file.name, src_embed_file.name,
+                    "--tgt_embed", tgt_overlap_file.name, tgt_embed_file.name,
+                ],
+                stdout=subprocess.PIPE,
+                text=True,
+            )
+
+            output_lines = result.stdout.strip().split("\n")
+            avg_cost, zero_cost_ratio = compute_alignment_stats(output_lines)
+            print(f"del_percentile_frac: {del_percentile_frac:.3f} | Avg Cost: {avg_cost:.6f} | Zero-Cost Ratio: {zero_cost_ratio:.2%}")
+
+            if first_flag:
+                first_flag = False
+
+            if prev_zero_cost_ratio is not None and prev_zero_cost_ratio != 0 and (zero_cost_ratio / prev_zero_cost_ratio) > 1.5:
+                print(f"Stopping exploration: Zero-cost ratio increased sharply at {del_percentile_frac:.3f}")
+                break
+            elif prev_zero_cost_ratio is not None and (
+                (zero_cost_ratio - prev_zero_cost_ratio) > 0.15 or
+                avg_cost > prev_avg_cost or
+                avg_cost < 0.3 or zero_cost_ratio > 0.7
+            ):
+                print(f"Stopping exploration: Zero-cost ratio increased sharply at {del_percentile_frac:.3f}")
+                break
+            else:
+                if avg_cost < best_avg_cost:
+                    best_avg_cost = avg_cost
+                    best_del_percentile_frac = del_percentile_frac
+                    best_zero_cost_ratio = zero_cost_ratio
+                    best_alignments = output_lines
+
+            prev_zero_cost_ratio = zero_cost_ratio
+            prev_avg_cost = avg_cost
+            del_percentile_frac -= step_size
+
+    # Parse the best alignments
+    parsed_alignments = []
+    for line in best_alignments:
+        if line:
+            src_part, tgt_part, _ = line.split(":")
+            src_indices = list(map(int, src_part.strip("[]").split(","))) if src_part.strip("[]") else []
+            tgt_indices = list(map(int, tgt_part.strip("[]").split(","))) if tgt_part.strip("[]") else []
+            parsed_alignments.append((src_indices, tgt_indices))
+
+    print("\nBest Found:")
+    print(f"del_percentile_frac: {best_del_percentile_frac:.3f} | Avg Cost: {best_avg_cost:.6f} | Zero-Cost Ratio: {best_zero_cost_ratio:.2%}")
+    return parsed_alignments
+
+
+def clean_sentence(sentence: str) -> str:
+    """
+    Clean a sentence by removing duplicate parts and reconnecting with the separator.
+
+    Args:
+        sentence (str): Input sentence.
+
+    Returns:
+        str: Cleaned sentence.
+    """
+    if not sentence:
+        return ""
+    parts = sentence.split(SEPARATOR)
+    unique_parts = list(dict.fromkeys(part.strip() for part in parts if part.strip()))
+    return f" {SEPARATOR} ".join(unique_parts) + f" {SEPARATOR}"
+
+
+def sliding_windows(sentences: list, window_size: int) -> list:
+    """
+    Create sliding windows from a list of sentences.
+
+    Args:
+        sentences (list): List of sentences.
+        window_size (int): Window size.
+
+    Returns:
+        list: List of sliding windows (each is a list of sentences).
+    """
+    windows = []
+    for i in range(len(sentences) - window_size + 1):
+        window = [clean_sentence(s) for s in sentences[i:i + window_size]]
+        # Remove duplicate window contents
+        unique_window = list(dict.fromkeys(window))
+        windows.append(unique_window)
+    return windows
+
+
+def save_windows_to_file(paragraph_id: int, aligned_src: list, aligned_ref: list, aligned_mt: list,
+                         src_windows: list, ref_windows: list, mt_windows: list,
+                         qe_src_windows: list, qe_mt_windows: list, output_dir: str,
+                         output_name: str) -> None:
+    """
+    Save window information and alignment data as JSON files.
+
+    Args:
+        paragraph_id (int): Paragraph ID.
+        aligned_src (list): Adjusted source alignment.
+        aligned_ref (list): Adjusted reference alignment.
+        aligned_mt (list): Adjusted MT alignment.
+        src_windows (list): Source sliding windows.
+        ref_windows (list): Reference sliding windows.
+        mt_windows (list): MT sliding windows.
+        qe_src_windows (list): QE source sliding windows.
+        qe_mt_windows (list): QE MT sliding windows.
+        output_dir (str): Output directory path.
+        output_name (str): Identifier for the output file.
+    """
+    os.makedirs(output_dir, exist_ok=True)
+
+    windows_data = {
+        "paragraph_id": paragraph_id,
+        "src_windows": src_windows,
+        "ref_windows": ref_windows,
+        "mt_windows": mt_windows,
+    }
+    windows_file = os.path.join(output_dir, f"windows_{paragraph_id}_{output_name}.json")
+    with open(windows_file, "w", encoding="utf-8") as f:
+        json.dump(windows_data, f, ensure_ascii=False, indent=2)
+
+    qe_windows_data = {
+        "paragraph_id": paragraph_id,
+        "src_windows": qe_src_windows,
+        "mt_windows": qe_mt_windows,
+    }
+    qe_windows_file = os.path.join(output_dir, f"qe_windows_{paragraph_id}_{output_name}.json")
+    with open(qe_windows_file, "w", encoding="utf-8") as f:
+        json.dump(qe_windows_data, f, ensure_ascii=False, indent=2)
+
+    aligned_info = {
+        "src": aligned_src,
+        "ref": aligned_ref,
+        "mt": aligned_mt,
+    }
+    aligned_file = os.path.join(output_dir, f"aligned_{paragraph_id}_{output_name}.json")
+    with open(aligned_file, "w", encoding="utf-8") as f:
+        json.dump(aligned_info, f, ensure_ascii=False, indent=2)
+
+
+# -----------------------------------------------------------------------------
+# Alignment Gap Processing Functions
+# -----------------------------------------------------------------------------
+def process_gaps(alignments: list) -> tuple:
+    """
+    Process alignment list blocks where the source is empty but target is non-empty,
+    converting them into gap alignments (source converted to a negative gap key).
+
+    Args:
+        alignments (list): Original alignment list (each element is (src_indices, tgt_indices)).
+
+    Returns:
+        tuple: (new_alignments (list), gap_counts (dict))
+    """
+    new_alignments = []
+    gap_counts = {}
+    n = len(alignments)
+    i = 0
+    while i < n:
+        src, tgt = alignments[i]
+        if not src and tgt:
+            block = []
+            while i < n and not alignments[i][0] and alignments[i][1]:
+                block.append(alignments[i])
+                i += 1
+            # Get the left neighbor's source index if available
+            left_src = new_alignments[-1][0][-1] if new_alignments and new_alignments[-1][0] else None
+            # Get the first non-empty source index on the right
+            right_src = None
+            j = i
+            while j < n:
+                if alignments[j][0]:
+                    right_src = alignments[j][0][0]
+                    break
+                j += 1
+            gap_key = left_src if left_src is not None else (right_src - 1 if right_src is not None else 0)
+            for item in block:
+                new_alignments.append(([-gap_key], item[1]))
+            gap_counts[gap_key] = gap_counts.get(gap_key, 0) + len(block)
+        else:
+            new_alignments.append(alignments[i])
+            i += 1
+    return new_alignments, gap_counts
+
+
+def complement_gaps(processed: list, gap_counts: dict, desired_gaps: dict) -> list:
+    """
+    Complement the gaps in the processed alignment list by inserting dummy alignments until
+    the desired gap count is met.
+
+    Args:
+        processed (list): Processed alignment list.
+        gap_counts (dict): Counts of each gap key in the processed list.
+        desired_gaps (dict): Desired counts for each gap key from the other alignment list.
+
+    Returns:
+        list: Processed alignment list after gap completion.
+    """
+    all_keys = set(gap_counts.keys()) | set(desired_gaps.keys())
+    for gap in all_keys:
+        current = gap_counts.get(gap, 0)
+        desired = desired_gaps.get(gap, 0)
+        if current < desired:
+            indices = [i for i, (src, _) in enumerate(processed) if src and src[0] == -gap]
+            insert_idx = indices[0] if indices else next((i for i, (src, _) in enumerate(processed) if src and src[0] > gap), len(processed))
+            for _ in range(desired - current):
+                processed.insert(insert_idx, ([-gap], []))
+            gap_counts[gap] = desired
+    return processed
+
+
+def custom_sort_key(item: tuple) -> tuple:
+    """
+    Custom sort key:
+      - For non-gap alignments (positive), key = (source, 0).
+      - For gap alignments (negative), key = (abs(source), 1).
+
+    Args:
+        item (tuple): Alignment tuple (src_indices, tgt_indices).
+
+    Returns:
+        tuple: Sorting key.
+    """
+    src, _ = item
+    if src:
+        val = src[0]
+        return (val, 0) if val >= 0 else (abs(val), 1)
+    return (float('inf'), 2)
+
+
+def fill_empty_alignments(src_ref_alignments: list, src_mt_alignments: list) -> tuple:
+    """
+    Fill the empty alignments (gaps) in both source-reference and source-MT alignments so that
+    the gap key counts match, then sort them.
+
+    Args:
+        src_ref_alignments (list): Alignment list for source-reference.
+        src_mt_alignments (list): Alignment list for source-MT.
+
+    Returns:
+        tuple: (filled_src_ref_alignments, filled_src_mt_alignments)
+    """
+    proc_ref, gaps_ref = process_gaps(src_ref_alignments)
+    proc_mt, gaps_mt = process_gaps(src_mt_alignments)
+    proc_ref = complement_gaps(proc_ref, gaps_ref, gaps_mt)
+    proc_mt = complement_gaps(proc_mt, gaps_mt, gaps_ref)
+    proc_ref.sort(key=custom_sort_key)
+    proc_mt.sort(key=custom_sort_key)
+    return proc_ref, proc_mt
+
+
+def find_common_alignments(src_ref_alignments: list, src_mt_alignments: list) -> list:
+    """
+    Find common alignments between source-reference and source-MT alignment lists and remove duplicates.
+
+    Args:
+        src_ref_alignments (list): Alignment list for source-reference.
+        src_mt_alignments (list): Alignment list for source-MT.
+
+    Returns:
+        list: List of common alignments as (common_src_indices, common_ref_indices, common_mt_indices).
+    """
+    common_alignments = []
+    src_ref_alignments, src_mt_alignments = fill_empty_alignments(src_ref_alignments, src_mt_alignments)
+
+    for ref_align in src_ref_alignments:
+        for mt_align in src_mt_alignments:
+            common_src = sorted(list(set(ref_align[0]) & set(mt_align[0])))
+            if common_src:
+                common_ref = sorted(list(set(ref_align[1]))) if ref_align[1] else [-1]
+                common_mt = sorted(list(set(mt_align[1]))) if mt_align[1] else [-1]
+                common_alignments.append((common_src, common_ref, common_mt))
+
+    # Remove duplicate triples
+    unique = []
+    seen = set()
+    for triple in common_alignments:
+        key = (tuple(triple[0]), tuple(triple[1]), tuple(triple[2]))
+        if key not in seen:
+            seen.add(key)
+            unique.append(triple)
+    print("Unique common alignments:")
+    print(unique)
+    return unique
+
+
+def args_to_dict(args: argparse.Namespace, prefix: str, strip_prefix: bool = False) -> dict:
+    """
+    Convert an argparse Namespace to a dictionary, optionally filtering by a prefix and stripping it.
+
+    Args:
+        args (argparse.Namespace): Input arguments.
+        prefix (str): Prefix to filter keys.
+        strip_prefix (bool): Whether to remove the prefix from keys (default is False).
+
+    Returns:
+        dict: Filtered dictionary.
+    """
+    d = vars(args)
+    prefix_key = prefix + '_'
+    filtered = {k: v for k, v in d.items() if k.startswith(prefix_key)}
+    if strip_prefix:
+        return {k[len(prefix_key):]: v for k, v in filtered.items()}
+    return filtered
+
+
+# -----------------------------------------------------------------------------
+# Metrics Computation
+# -----------------------------------------------------------------------------
+def compute_metrics(paragraph_src: str, paragraph_ref: str, paragraph_mt: str,
+                    src_windows: list, ref_windows: list, mt_windows: list,
+                    qe_src_windows: list, qe_mt_windows: list,
+                    paragraph_id: int, mt_col: str) -> dict:
+    """
+    Compute COMET and COMET-QE scores, then save the scores and related window information as a JSON file.
+
+    Args:
+        paragraph_src (str): Source paragraph text.
+        paragraph_ref (str): Reference paragraph text.
+        paragraph_mt (str): MT paragraph text.
+        bleu_adjusted_ref: (Placeholder) BLEU adjusted parameter.
+        bleu_adjusted_mt: (Placeholder) BLEU adjusted parameter.
+        src_windows (list): Source sliding windows.
+        ref_windows (list): Reference sliding windows.
+        mt_windows (list): MT sliding windows.
+        qe_src_windows (list): QE source sliding windows.
+        qe_mt_windows (list): QE MT sliding windows.
+        paragraph_id (int): Paragraph ID.
+        mt_col (str): MT column name.
+
+    Returns:
+        dict: Dictionary containing various computed scores.
+    """
+    comet_zero_score_windows = []
+    comet_qe_zero_score_windows = []
+
+    with tempfile.NamedTemporaryFile(mode='w+', delete=True) as src_file, \
+         tempfile.NamedTemporaryFile(mode='w+', delete=True) as ref_file, \
+         tempfile.NamedTemporaryFile(mode='w+', delete=True) as mt_file, \
+         tempfile.NamedTemporaryFile(mode='w+', delete=True) as qe_src_file, \
+         tempfile.NamedTemporaryFile(mode='w+', delete=True) as qe_mt_file:
+
+        # Write each window on a separate line
+        for idx, (src_win, ref_win, mt_win) in enumerate(zip(src_windows, ref_windows, mt_windows)):
+            src_line = " ".join(src_win)
+            ref_line = " ".join(ref_win)
+            mt_line = " ".join(mt_win)
+            if src_line and mt_line:
+                src_file.write(src_line + "\n")
+                ref_file.write(ref_line + "\n")
+                mt_file.write(mt_line + "\n")
+            else:
+                comet_zero_score_windows.append(idx)
+
+        src_file.flush()
+        ref_file.flush()
+        mt_file.flush()
+
+        comet_command = [
+            "comet-score",
+            "-s", src_file.name,
+            "-t", mt_file.name,
+            "-r", ref_file.name,
+            "--model", COMET_MODEL,
+            "--enable-context",
+            "--gpus", GPU_ID,
+            "--quiet",
+        ]
+        result = subprocess.run(comet_command, stdout=subprocess.PIPE, text=True)
+        print(result.stdout)
+        comet_scores = [float(s) for s in re.findall(r"score:\s(-?[0-9.]+)", result.stdout.strip())][:-1]
+
+        for idx, (src_win, mt_win) in enumerate(zip(qe_src_windows, qe_mt_windows)):
+            src_line = " ".join(src_win)
+            mt_line = " ".join(mt_win)
+            if src_line and mt_line:
+                qe_src_file.write(src_line + "\n")
+                qe_mt_file.write(mt_line + "\n")
+            else:
+                comet_qe_zero_score_windows.append(idx)
+
+        qe_src_file.flush()
+        qe_mt_file.flush()
+
+        qe_command = [
+            "comet-score",
+            "-s", qe_src_file.name,
+            "-t", qe_mt_file.name,
+            "--model", COMET_QE_MODEL,
+            "--enable-context",
+            "--gpus", GPU_ID,
+            "--quiet",
+        ]
+        qe_result = subprocess.run(qe_command, stdout=subprocess.PIPE, text=True)
+        print(qe_result.stdout)
+        comet_qe_scores = [float(s) for s in re.findall(r"score:\s(-?[0-9.]+)", qe_result.stdout.strip())][:-1]
+
+    # Insert zero scores for windows that had missing scores
+    for idx in comet_zero_score_windows:
+        comet_scores.insert(idx, 0.0)
+    for idx in comet_qe_zero_score_windows:
+        comet_qe_scores.insert(idx, 0.0)
+
+    # Placeholder values for sentence-level metrics
+    sentences_length = len(paragraph_mt.splitlines())
+    sentences_zero_ratio = 0.0
+
+    scores_data = {
+        'paragraph_id': paragraph_id,
+        'comet_scores': comet_scores,
+        'comet_qe_scores': comet_qe_scores,
+        'sentences_length': sentences_length,
+        'windows_length': len(comet_scores),
+        'windows_qe_length': len(comet_qe_scores),
+        'sentences_zero_ratio': sentences_zero_ratio,
+        'windows_zero_ratio': len(comet_zero_score_windows) / len(comet_scores) if comet_scores else 0,
+        'windows_qe_zero_ratio': len(comet_qe_zero_score_windows) / len(comet_qe_scores) if comet_qe_scores else 0,
+        'avg_comet': sum(comet_scores) / len(comet_scores) if comet_scores else 0,
+        'avg_comet_qe': sum(comet_qe_scores) / len(comet_qe_scores) if comet_qe_scores else 0
+    }
+
+    scores_file = os.path.join(SAVE_FOLDER, 'scores', f'scores_{paragraph_id}_{mt_col}.json')
+    os.makedirs(os.path.dirname(scores_file), exist_ok=True)
+    with open(scores_file, 'w', encoding='utf-8') as f:
+        json.dump(scores_data, f, ensure_ascii=False, indent=2)
+
+    return scores_data
+
+
+def compute_metrics_reference_free(src_windows: list, mt_windows: list,
+                                   qe_src_windows: list, qe_mt_windows: list,
+                                   paragraph_id: int, mt_col: str) -> dict:
+    """
+    Compute reference-free evaluation metrics (only QE scores) when no reference is provided.
+
+    Args:
+        src_windows (list): (Unused) Source sliding windows.
+        mt_windows (list): (Unused) MT sliding windows.
+        qe_src_windows (list): QE source sliding windows.
+        qe_mt_windows (list): QE MT sliding windows.
+        paragraph_id (int): Paragraph ID.
+        mt_col (str): MT column name.
+
+    Returns:
+        dict: Dictionary containing computed QE scores.
+    """
+    comet_qe_zero_score_windows = []
+
+    with tempfile.NamedTemporaryFile(mode='w+', delete=True) as qe_src_file, \
+         tempfile.NamedTemporaryFile(mode='w+', delete=True) as qe_mt_file:
+
+        for idx, (src_win, mt_win) in enumerate(zip(qe_src_windows, qe_mt_windows)):
+            src_line = " ".join(src_win)
+            mt_line = " ".join(mt_win)
+            if src_line and mt_line:
+                qe_src_file.write(src_line + "\n")
+                qe_mt_file.write(mt_line + "\n")
+            else:
+                comet_qe_zero_score_windows.append(idx)
+        qe_src_file.flush()
+        qe_mt_file.flush()
+
+        qe_command = [
+            "comet-score",
+            "-s", qe_src_file.name,
+            "-t", qe_mt_file.name,
+            "--model", COMET_QE_MODEL,
+            "--enable-context",
+            "--gpus", GPU_ID,
+            "--quiet",
+        ]
+        qe_result = subprocess.run(qe_command, stdout=subprocess.PIPE, text=True)
+        print(qe_result.stdout)
+        comet_qe_scores = [float(s) for s in re.findall(r"score:\s(-?[0-9.]+)", qe_result.stdout.strip())][:-1]
+
+    for idx in comet_qe_zero_score_windows:
+        comet_qe_scores.insert(idx, 0.0)
+
+    scores_data = {
+        'paragraph_id': paragraph_id,
+        'comet_scores': 0.0,  # Not computed in reference-free mode.
+        'comet_qe_scores': comet_qe_scores,
+        'windows_length': len(comet_qe_scores),
+        'windows_qe_length': len(comet_qe_scores),
+        'avg_comet': 0.0,
+        'avg_comet_qe': sum(comet_qe_scores) / len(comet_qe_scores) if comet_qe_scores else 0,
+    }
+    return scores_data
+
+
+# -----------------------------------------------------------------------------
+# Paragraph-Level Processing
+# -----------------------------------------------------------------------------
+def paragraph_level_score(row: pd.Series, paragraph_id: int, src_col: str = None,
+                          ref_col: str = None, mt_col: str = None) -> None:
+    """
+    Process alignment and scoring for a single paragraph. Steps include:
+      1. Sentence segmentation and preprocessing.
+      2. Generating overlaps and embeddings.
+      3. Running vector alignment exploration.
+      4. Computing COMET and COMET-QE scores and saving window information.
+
+    Args:
+        row (pd.Series): A single data row.
+        paragraph_id (int): Paragraph identifier.
+        src_col (str): Source column name (default is "zh").
+        ref_col (str): Reference column name (default is set based on language).
+        mt_col (str): MT column name (default is set based on TARGET).
+    """
+    global mt_nlp, src_nlp
+
+    # Set default columns if not provided
+    if ref_col is None:
+        ref_col = LANG
+    if mt_col is None:
+        mt_col = TARGET
+
+    # Sentence segmentation and preprocessing
+    src_sentences = segment_sentences_by_punctuation(row[src_col], src_col)
+    ref_sentences = segment_sentences_by_punctuation(row[ref_col], ref_col)
+    mt_sentences = segment_sentences_by_punctuation(row[mt_col], ref_col)
+
+    src_txt = preprocess_sentences(src_sentences)
+    ref_txt = preprocess_sentences(ref_sentences)
+    mt_txt = preprocess_sentences(mt_sentences)
+
+    # Generate overlap and embedding data
+    src_overlap, src_embed = generate_overlap_and_embedding(src_txt)
+    ref_overlap, ref_embed = generate_overlap_and_embedding(ref_txt)
+    mt_overlap, mt_embed = generate_overlap_and_embedding(mt_txt)
+
+    # Run vector alignment exploration
+    src_ref_alignments = run_vecalign_explore(src_txt, ref_txt, src_overlap, ref_overlap, src_embed, ref_embed)
+    src_mt_alignments = run_vecalign_explore(src_txt, mt_txt, src_overlap, mt_overlap, src_embed, mt_embed)
+
+    # For reference-free evaluation: get non-adjusted alignments
+    non_adjusted_src = []
+    non_adjusted_mt = []
+    for src_indices, mt_indices in src_mt_alignments:
+        mt_indices = [x for x in mt_indices if x != -1]
+        aligned_src = " ".join([src_sentences[i] for i in src_indices]) if src_indices else ""
+        aligned_mt = " ".join([mt_sentences[i] for i in mt_indices]) if mt_indices else ""
+        non_adjusted_src.append(aligned_src)
+        non_adjusted_mt.append(aligned_mt)
+
+    # Find common alignments between src-ref and src-mt
+    common_alignments = find_common_alignments(src_ref_alignments, src_mt_alignments)
+
+    adjusted_src, adjusted_ref, adjusted_mt = [], [], []
+    for src_indices, ref_indices, mt_indices in common_alignments:
+        ref_indices = [x for x in ref_indices if x != -1]
+        mt_indices = [x for x in mt_indices if x != -1]
+        aligned_src = "" if (src_indices and src_indices[0] < 0) else " ".join([src_sentences[i] for i in src_indices])
+        aligned_ref = " ".join([ref_sentences[i] for i in ref_indices]) if ref_indices else ""
+        aligned_mt = " ".join([mt_sentences[i] for i in mt_indices]) if mt_indices else ""
+        adjusted_src.append(aligned_src)
+        adjusted_ref.append(aligned_ref)
+        adjusted_mt.append(aligned_mt)
+
+    # Create sliding windows
+    src_windows = sliding_windows(adjusted_src, WINDOW_SIZE)
+    ref_windows = sliding_windows(adjusted_ref, WINDOW_SIZE)
+    mt_windows = sliding_windows(adjusted_mt, WINDOW_SIZE)
+    qe_src_windows = sliding_windows(non_adjusted_src, WINDOW_SIZE)
+    qe_mt_windows = sliding_windows(non_adjusted_mt, WINDOW_SIZE)
+
+    # Compute metrics and save window information
+    compute_metrics(
+        row[src_col], row[ref_col], row[mt_col],
+        src_windows, ref_windows, mt_windows,
+        qe_src_windows, qe_mt_windows,
+        paragraph_id, mt_col
+    )
+
+    output_dir = os.path.join(SAVE_FOLDER, "windows")
+    save_windows_to_file(paragraph_id, adjusted_src, adjusted_ref, adjusted_mt,
+                         src_windows, ref_windows, mt_windows,
+                         qe_src_windows, qe_mt_windows, output_dir, output_name=mt_col)
+
+
+def parallel_paragraph_level_score(args: tuple) -> None:
+    """
+    Process a single paragraph in parallel. If an exception occurs, it prints an error message.
+
+    Args:
+        args (tuple): (row (pd.Series), paragraph_id (int))
+    """
+    row, paragraph_id = args
+    try:
+        paragraph_level_score(row, paragraph_id, mt_col=TARGET, src_col= SRC_LANG)
+    except Exception as e:
+        print(f"Error processing paragraph {paragraph_id}: {e}")
+        print(f"{TARGET} result cannot be aligned in paragraph {paragraph_id}\n")
+
+
+# -----------------------------------------------------------------------------
+# New Function: Flexible Evaluation (Reference-Free or Full Evaluation)
+# -----------------------------------------------------------------------------
+def evaluate_score(src: str, tgt: str, ref: Optional[str] = None) -> dict:
+    """
+    Evaluate quality scores for given source and target texts.
+    If a reference is provided, full evaluation is performed (including src-ref alignment);
+    otherwise, reference-free evaluation is conducted using only src and tgt.
+
+    Args:
+        src (str): Source text.
+        tgt (str): Target (MT) text.
+        ref (Optional[str]): Reference text (if provided).
+
+    Returns:
+        dict: Dictionary of evaluation scores.
+    """
+    # Full evaluation (with reference)
+    if ref is not None:
+        src_sentences = segment_sentences_by_punctuation(src, SRC_LANG)
+        ref_sentences = segment_sentences_by_punctuation(ref, LANG)
+        tgt_sentences = segment_sentences_by_punctuation(tgt, LANG)
+
+        src_txt = preprocess_sentences(src_sentences)
+        ref_txt = preprocess_sentences(ref_sentences)
+        tgt_txt = preprocess_sentences(tgt_sentences)
+
+        src_overlap, src_embed = generate_overlap_and_embedding(src_txt)
+        ref_overlap, ref_embed = generate_overlap_and_embedding(ref_txt)
+        tgt_overlap, tgt_embed = generate_overlap_and_embedding(tgt_txt)
+
+        src_ref_alignments = run_vecalign_explore(src_txt, ref_txt, src_overlap, ref_overlap, src_embed, ref_embed)
+        src_mt_alignments = run_vecalign_explore(src_txt, tgt_txt, src_overlap, tgt_overlap, src_embed, tgt_embed)
+
+        non_adjusted_src = []
+        non_adjusted_mt = []
+        for s_indices, t_indices in src_mt_alignments:
+            filtered_t_indices = [x for x in t_indices if x != -1]
+            aligned_src = " ".join([src_sentences[i] for i in s_indices]) if s_indices else ""
+            aligned_mt = " ".join([tgt_sentences[i] for i in filtered_t_indices]) if filtered_t_indices else ""
+            non_adjusted_src.append(aligned_src)
+            non_adjusted_mt.append(aligned_mt)
+
+        common_alignments = find_common_alignments(src_ref_alignments, src_mt_alignments)
+        adjusted_src, adjusted_ref, adjusted_mt = [], [], []
+        for s_indices, r_indices, t_indices in common_alignments:
+            r_indices = [x for x in r_indices if x != -1]
+            t_indices = [x for x in t_indices if x != -1]
+            aligned_src = "" if (s_indices and s_indices[0] < 0) else " ".join([src_sentences[i] for i in s_indices])
+            aligned_ref = " ".join([ref_sentences[i] for i in r_indices]) if r_indices else ""
+            aligned_mt = " ".join([tgt_sentences[i] for i in t_indices]) if t_indices else ""
+            adjusted_src.append(aligned_src)
+            adjusted_ref.append(aligned_ref)
+            adjusted_mt.append(aligned_mt)
+
+        src_windows = sliding_windows(adjusted_src, WINDOW_SIZE)
+        ref_windows = sliding_windows(adjusted_ref, WINDOW_SIZE)
+        tgt_windows = sliding_windows(adjusted_mt, WINDOW_SIZE)
+        qe_src_windows = sliding_windows(non_adjusted_src, WINDOW_SIZE)
+        qe_mt_windows = sliding_windows(non_adjusted_mt, WINDOW_SIZE)
+
+        # Use paragraph_id=0 for single evaluation
+        scores_data = compute_metrics(src, ref, tgt,
+                                      src_windows, ref_windows, tgt_windows,
+                                      qe_src_windows, qe_mt_windows,
+                                      paragraph_id=0, mt_col=TARGET)
+        return scores_data
+
+    # Reference-free evaluation
+    else:
+        src_sentences = segment_sentences_by_punctuation(src, SRC_LANG)
+        tgt_sentences = segment_sentences_by_punctuation(tgt, LANG)
+
+        src_txt = preprocess_sentences(src_sentences)
+        tgt_txt = preprocess_sentences(tgt_sentences)
+
+        src_overlap, src_embed = generate_overlap_and_embedding(src_txt)
+        tgt_overlap, tgt_embed = generate_overlap_and_embedding(tgt_txt)
+
+        src_mt_alignments = run_vecalign_explore(src_txt, tgt_txt, src_overlap, tgt_overlap, src_embed, tgt_embed)
+
+        non_adjusted_src = []
+        non_adjusted_mt = []
+        for s_indices, t_indices in src_mt_alignments:
+            filtered_t_indices = [x for x in t_indices if x != -1]
+            aligned_src = " ".join([src_sentences[i] for i in s_indices]) if s_indices else ""
+            aligned_mt = " ".join([tgt_sentences[i] for i in filtered_t_indices]) if filtered_t_indices else ""
+            non_adjusted_src.append(aligned_src)
+            non_adjusted_mt.append(aligned_mt)
+
+        # In reference-free mode, only compute QE evaluation.
+        qe_src_windows = sliding_windows(non_adjusted_src, WINDOW_SIZE)
+        qe_mt_windows = sliding_windows(non_adjusted_mt, WINDOW_SIZE)
+
+        scores_data = compute_metrics_reference_free(src_windows=[], mt_windows=[],
+                                                     qe_src_windows=qe_src_windows, qe_mt_windows=qe_mt_windows,
+                                                     paragraph_id=0, mt_col=TARGET)
+        return scores_data
+
+
+def aggregate_scores_and_merge(evaluated_file_path: str, save_folder: str, target: str) -> dict:
+    """
+    Read scores for each paragraph, aggregate the results, and save them as a CSV.
+
+    Args:
+        evaluated_file_path (str): Path to the original CSV file.
+        save_folder (str): Folder where scores are saved.
+        target (str): MT target name.
+
+    Returns:
+        dict: Overall average scores for each metric.
+    """
+    df = pd.read_csv(evaluated_file_path)
+    df['comet'] = 0.0
+    df['comet_qe'] = 0.0
+    df['sentences_zero_ratio'] = 0.0
+    df['windows_zero_ratio'] = 0.0
+    df['windows_qe_zero_ratio'] = 0.0
+
+    scores_dir = os.path.join(save_folder, 'scores')
+    total_scores = {
+        'comet': [],
+        'comet_qe': [],
+        'sentences_zero_ratio': [],
+        'windows_zero_ratio': [],
+        'windows_qe_zero_ratio': []
+    }
+
+    for idx in df.index:
+        score_file = os.path.join(scores_dir, f'scores_{idx}_{target}.json')
+        if os.path.exists(score_file):
+            with open(score_file, 'r', encoding='utf-8') as f:
+                scores = json.load(f)
+                df.at[idx, 'comet'] = scores.get('avg_comet', 0)
+                df.at[idx, 'comet_qe'] = scores.get('avg_comet_qe', 0)
+                df.at[idx, 'sentences_zero_ratio'] = scores.get('sentences_zero_ratio', 0)
+                df.at[idx, 'windows_zero_ratio'] = scores.get('windows_zero_ratio', 0)
+                df.at[idx, 'windows_qe_zero_ratio'] = scores.get('windows_qe_zero_ratio', 0)
+
+                total_scores['comet'].append(scores.get('avg_comet', 0))
+                total_scores['comet_qe'].append(scores.get('avg_comet_qe', 0))
+                total_scores['sentences_zero_ratio'].append(scores.get('sentences_zero_ratio', 0))
+                total_scores['windows_zero_ratio'].append(scores.get('windows_zero_ratio', 0))
+                total_scores['windows_qe_zero_ratio'].append(scores.get('windows_qe_zero_ratio', 0))
+
+    overall_scores = {metric: (sum(vals) / len(vals) if vals else 0) for metric, vals in total_scores.items()}
+
+    output_path = os.path.join(save_folder, f'evaluated_results_{target}.csv')
+    df.to_csv(output_path, index=False)
+    return overall_scores
+
+
+# -----------------------------------------------------------------------------
+# Global Parameters
+# -----------------------------------------------------------------------------
+set_seed(42)
+
+# Set up argparse with defaults for file, target_column, and save folder.
+parser = argparse.ArgumentParser(description="Set TARGET_FILE, TARGET_COLUMN, and TASK_LANGUAGE")
+parser.add_argument("--file", type=str, default="", help="(Optional) Set the MT target file")
+parser.add_argument("--target_column", type=str, default="", help="(Optional) Set the MT target column")
+parser.add_argument("--save", type=str, default="./", help="(Optional) Set the save folder")
+parser.add_argument("--src_language", type=str, required=True, help="Set the task language (English, Russian, German)")
+parser.add_argument("--task_language", type=str, required=True, help="Set the task language (English, Russian, German)")
+
+args = parser.parse_args()
+
+TARGET = args.target_column
+TASK_LANGUAGE = args.task_language
+SRC_LANGUAGE = args.src_language
+print(f"TARGET: {TARGET}")
+print(f"TASK_LANGUAGE: {TASK_LANGUAGE}")
+
+if TASK_LANGUAGE == "English":
+    LANG = 'en'
+elif TASK_LANGUAGE == "Russian":
+    LANG = 'ru'
+elif TASK_LANGUAGE == "German":
+    LANG = 'de'
+elif TASK_LANGUAGE == "Japanese":
+    LANG = 'ja'
+elif TASK_LANGUAGE == "Spanish":
+    LANG = 'es'
+elif TASK_LANGUAGE == "Chinese":
+    LANG = 'zh'
+else:
+    raise ValueError("Unsupported TASK_LANGUAGE.")
+
+if SRC_LANGUAGE == "English":
+    SRC_LANG = 'en'
+elif SRC_LANGUAGE == "Russian":
+    SRC_LANG = 'ru'
+elif SRC_LANGUAGE == "German":
+    SRC_LANG = 'de'
+elif SRC_LANGUAGE == "Japanese":
+    SRC_LANG = 'ja'
+elif SRC_LANGUAGE == "Spanish":
+    SRC_LANG = 'es'
+elif SRC_LANGUAGE == "Chinese":
+    SRC_LANG = 'zh'
+else:
+    raise ValueError("Unsupported TASK_LANGUAGE.")
+
+# File and folder path settings
+evaluated_file_path = args.file  # May be empty if not provided
+WINDOW_SIZE = 3
+SEPARATOR = "</s>"
+SAVE_FOLDER = args.save
+GPU_ID = "1"
+COMET_MODEL = "Unbabel/wmt22-comet-da"
+COMET_QE_MODEL = "Unbabel/wmt22-cometkiwi-da"
+
+if not os.path.exists(SAVE_FOLDER):
+    os.makedirs(SAVE_FOLDER)
+    print(f"Folder '{SAVE_FOLDER}' created")
+else:
+    print(f"Folder '{SAVE_FOLDER}' already exists")
+
+# Load Spacy models based on task language
+if TASK_LANGUAGE == "English":
+    mt_nlp = spacy.load("en_core_web_sm")
+elif TASK_LANGUAGE == "Russian":
+    mt_nlp = spacy.load("ru_core_news_sm")
+elif TASK_LANGUAGE == "German":
+    mt_nlp = spacy.load("de_core_news_sm")
+elif TASK_LANGUAGE == "Japanese":
+    mt_nlp = spacy.load("ja_core_news_sm")
+elif TASK_LANGUAGE == "Spanish":
+    mt_nlp = spacy.load("es_core_news_sm")
+elif TASK_LANGUAGE == "Chinese":
+    mt_nlp = spacy.load("zh_core_web_sm")
+    
+if SRC_LANGUAGE == "English":
+    src_nlp = spacy.load("en_core_web_sm")
+elif SRC_LANGUAGE == "Russian":
+    src_nlp = spacy.load("ru_core_news_sm")
+elif SRC_LANGUAGE == "German":
+    src_nlp = spacy.load("de_core_news_sm")
+elif SRC_LANGUAGE == "Japanese":
+    src_nlp = spacy.load("ja_core_news_sm")
+elif SRC_LANGUAGE == "Spanish":
+    src_nlp = spacy.load("es_core_news_sm")
+elif SRC_LANGUAGE == "Chinese":
+    src_nlp = spacy.load("zh_core_web_sm")
+
+# -----------------------------------------------------------------------------
+# Main Process: Parallel processing of paragraphs and score aggregation
+# Command: export LASER="/path/to/laser/"
+# Command for evaluate csv: 
+# python long_context_eval.py --file eval_en_ja.csv --target_column mpc --save eval_en_ja --src_language English --task_language Japanese
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    data = pd.read_csv(evaluated_file_path)
+    pool_args = [(row, idx) for idx, row in data.iterrows()]
+    with Pool(2) as pool:
+        pool.map(parallel_paragraph_level_score, pool_args)
+    overall_scores = aggregate_scores_and_merge(evaluated_file_path, SAVE_FOLDER, TARGET)
+    timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    output_result = f"{TARGET}: {TASK_LANGUAGE} Overall scores: {overall_scores}, time: {timestamp}\n"
+    print(output_result)

vecalign/memory2csv.py

@@ -0,0 +1,64 @@
+import os
+import json
+import pandas as pd
+import argparse
+
+def process_csv(num, input_csv, output_csv, column_name):
+    # Read the input CSV file
+    try:
+        df = pd.read_csv(input_csv)
+    except Exception as e:
+        print(f"Error reading {input_csv}: {e}")
+        return
+
+    # Add the target column
+    df[column_name] = None
+
+    # Process each row by using the index to construct the JSON file path
+    for idx, row in df.iterrows():
+        json_path = os.path.join(os.path.splitext(input_csv)[0], f"metadata_{idx}_iter_{num}.json")
+
+        # Check if the file exists
+        if not os.path.exists(json_path):
+            print(f"File not found: {json_path}")
+            df.at[idx, column_name] = None
+            continue
+
+        # Open and read the JSON file
+        try:
+            with open(json_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+        except Exception as e:
+            print(f"Error reading {json_path}: {e}")
+            df.at[idx, column_name] = None
+            continue
+
+        # Extract the value from final_translations_record
+        final_record = data.get("final_translations_record", [])
+        if isinstance(final_record, list) and len(final_record) > 0:
+            value = final_record[0]
+        else:
+            value = None
+
+        # Write the value into the target column
+        df.at[idx, column_name] = value
+
+    # Save the result to output CSV
+    try:
+        df.to_csv(output_csv, index=False)
+        print(f"Saved successfully: {output_csv}")
+    except Exception as e:
+        print(f"Error saving {output_csv}: {e}")
+
+
+# Example command: python memory2csv.py --num 5 --input_csv valid_en_ja.csv --output_csv eval_en_ja.csv --column_name mpc
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Process CSV and extract data from JSON files.')
+    parser.add_argument('--num', type=int, required=True, help='Iteration number used in JSON filenames')
+    parser.add_argument('--input_csv', type=str, required=True, help='Path to input CSV file')
+    parser.add_argument('--output_csv', type=str, required=True, help='Path to save the output CSV file')
+    parser.add_argument('--column_name', type=str, required=True, help='Column name to store extracted values')
+
+    args = parser.parse_args()
+
+    process_csv(args.num, args.input_csv, args.output_csv, args.column_name)

vecalign/metricx24/__init__.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

vecalign/metricx24/evaluate.py

@@ -0,0 +1,155 @@
+# coding=utf-8
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evaluates the predictions from a MetricX model."""
+
+import dataclasses
+import json
+import os
+from typing import Any, Tuple
+
+from mt_metrics_eval import data
+from mt_metrics_eval import stats
+from mt_metrics_eval import tau_optimization
+import numpy as np
+import scipy.stats
+import transformers
+
+
+@dataclasses.dataclass
+class Arguments:
+  dataset: str = dataclasses.field(metadata={"help": "The MTME dataset."})
+
+  lp: str = dataclasses.field(metadata={"help": "The language pair."})
+
+  input_file: str = dataclasses.field(metadata={"help": "The input file."})
+
+  output_file: str = dataclasses.field(
+      metadata={"help": "The output file with evaluation metrics."},
+  )
+
+
+def _convert_to_matrices(
+    instances: list[dict[str, Any]]
+) -> Tuple[np.ndarray, np.ndarray]:
+  """Converts the instances to metric and human score matrices."""
+  system_id_to_row = {}
+  segment_id_to_col = {}
+
+  for instance in instances:
+    system_id = instance["system_id"]
+    segment_id = instance["segment_id"]
+    if system_id not in system_id_to_row:
+      system_id_to_row[system_id] = len(system_id_to_row)
+    if segment_id not in segment_id_to_col:
+      segment_id_to_col[segment_id] = len(segment_id_to_col)
+
+  num_rows = len(system_id_to_row)
+  num_cols = len(segment_id_to_col)
+  # MTME requires that missing scores must be None, not NaN.
+  metric_scores = np.full((num_rows, num_cols), None, dtype=np.dtype(object))
+  human_scores = np.full((num_rows, num_cols), None, dtype=np.dtype(object))
+
+  for instance in instances:
+    system_id = instance["system_id"]
+    segment_id = instance["segment_id"]
+    row = system_id_to_row[system_id]
+    col = segment_id_to_col[segment_id]
+    metric_scores[row, col] = (
+        -1 * instance["prediction"]
+    )  # negate so higher is better
+    human_scores[row, col] = instance["label"]
+
+  return metric_scores, human_scores
+
+
+def main() -> None:
+  parser = transformers.HfArgumentParser(Arguments)
+  (args,) = parser.parse_args_into_dataclasses()
+
+  # Download MTME data
+  data.Download()
+
+  # Load the data and filter outliers, the human system corresponding to the
+  # references, and any system that doesn't have any MQM scores.
+  evs = data.EvalSet(args.dataset, args.lp)
+  bad_systems = {evs.std_ref} | evs.outlier_sys_names
+  mqm = evs.Scores("seg", "mqm")
+  for system_id, scores in mqm.items():
+    if not any(score is not None for score in scores):
+      bad_systems.add(system_id)
+
+  instances = []
+  with open(args.input_file, "r") as f:
+    for line in f:
+      instance = json.loads(line)
+      if instance["system_id"] in bad_systems:
+        continue
+      instances.append(instance)
+
+  metric_seg_scores, human_seg_scores = _convert_to_matrices(instances)
+  metric_sys_scores = np.mean(metric_seg_scores, axis=1)
+  human_sys_scores = np.apply_along_axis(
+      lambda row: np.mean(row[row != None]), 1, human_seg_scores  # pylint: disable=singleton-comparison
+  )
+
+  # Segment-level correlations.
+  mask = human_seg_scores.reshape(-1) != None  # pylint: disable=singleton-comparison
+  seg_no_grouping_pearson, _ = scipy.stats.pearsonr(
+      metric_seg_scores.reshape(-1)[mask],
+      human_seg_scores.reshape(-1)[mask],
+  )
+  tie_calib_result = tau_optimization.tau_optimization(
+      metric_seg_scores.T,
+      human_seg_scores.T,
+      tau_optimization.TauSufficientStats.acc_23,
+  )
+
+  # System-level correlations.
+  sys_pearson, _ = scipy.stats.pearsonr(human_sys_scores, metric_sys_scores)
+  agree, num_pairs = stats.Agreement(human_sys_scores, metric_sys_scores)
+  sys_accuracy = agree / num_pairs
+  sys_spa = stats.PairwiseConfidenceError(
+      human_seg_scores.reshape(-1),
+      metric_seg_scores.reshape(-1),
+      human_seg_scores.shape[0],
+      filter_nones=True,
+  )[0]
+
+  metrics = {
+      "system_level": {
+          "pearson": sys_pearson,
+          "accuracy": sys_accuracy,
+          "spa": sys_spa,
+      },
+      "segment_level_no_grouping": {
+          "pearson": seg_no_grouping_pearson,
+      },
+      "segment_level_group_by_item": {
+          "accuracy": tie_calib_result.best_tau,
+          "epsilon": tie_calib_result.best_threshold,
+      },
+  }
+  print(json.dumps(metrics, indent=2))
+
+  if args.output_file:
+    dirname = os.path.dirname(args.output_file)
+    if dirname:
+      os.makedirs(dirname, exist_ok=True)
+    with open(args.output_file, "w") as out:
+      out.write(json.dumps(metrics))
+
+
+if __name__ == "__main__":
+  main()

vecalign/metricx24/evaluate_wmt24.py

@@ -0,0 +1,116 @@
+# coding=utf-8
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evaluates the predictions from a MetricX model."""
+
+import collections
+import dataclasses
+import json
+import os
+
+from mt_metrics_eval import data
+from mt_metrics_eval import tasks
+import numpy as np
+import transformers
+
+
+@dataclasses.dataclass
+class Arguments:
+  en_de: str = dataclasses.field(metadata={"help": "The en-de input file."})
+  en_es: str = dataclasses.field(metadata={"help": "The en-es input file."})
+  ja_zh: str = dataclasses.field(metadata={"help": "The ja-zh input file."})
+
+  output_file: str = dataclasses.field(
+      metadata={"help": "The output file with evaluation metrics."},
+  )
+
+
+def _load_scores(
+    input_file: str, num_segments: int,
+) -> tuple[dict[str, list[float]], dict[str, list[float]]]:
+  """Loads segment and system-level scores."""
+  scores = collections.defaultdict(dict)
+  with open(input_file, "r") as f:
+    for line in f:
+      instance = json.loads(line)
+      system_id = instance["system_id"]
+      segment_id = instance["segment_id"]
+      score = -1 * instance["prediction"]
+      scores[system_id][segment_id] = score
+
+  seg_scores = {}
+  for system_id in scores:
+    seg_scores[system_id] = []
+    for segment_id in range(num_segments):
+      seg_scores[system_id].append(scores[system_id].get(segment_id, None))
+
+  sys_scores = {}
+  for system_id in seg_scores:
+    cur_scores = np.asarray(seg_scores[system_id])
+    sys_scores[system_id] = np.mean(cur_scores[cur_scores != None])  # pylint: disable=singleton-comparison
+
+  return seg_scores, sys_scores
+
+
+def main() -> None:
+  parser = transformers.HfArgumentParser(Arguments)
+  (args,) = parser.parse_args_into_dataclasses()
+
+  # Download MTME data
+  data.Download()
+
+  metric_name = "metricx-24-v2p6"
+  wmt24_lps = ["en-de", "en-es", "ja-zh"]
+  evs_dict = {
+      ("wmt24", lp): data.EvalSet("wmt24", lp, True) for lp in wmt24_lps
+  }
+
+  segment_counts_per_lp = {}
+  for lp in wmt24_lps:
+    evs = evs_dict[("wmt24", lp)]
+    gold_scores = evs.Scores("seg", "mqm")
+    for _, scores in gold_scores.items():
+      segment_counts_per_lp[lp] = len(scores)
+      continue
+  scores = {
+      "en-de": _load_scores(args.en_de, segment_counts_per_lp["en-de"]),
+      "en-es": _load_scores(args.en_es, segment_counts_per_lp["en-es"]),
+      "ja-zh": _load_scores(args.ja_zh, segment_counts_per_lp["ja-zh"]),
+  }
+
+  for lp in wmt24_lps:
+    evs = evs_dict[("wmt24", lp)]
+    seg_scores, sys_scores = scores[lp]
+    evs._scores["seg"][f"{metric_name}-{evs.std_ref}"] = seg_scores  # pylint: disable=protected-access
+    evs._scores["sys"][f"{metric_name}-{evs.std_ref}"] = sys_scores  # pylint: disable=protected-access
+    evs._metric_names.add(f"{metric_name}-{evs.std_ref}")  # pylint: disable=protected-access
+    evs._metric_basenames.add(metric_name)  # pylint: disable=protected-access
+
+  for evs in evs_dict.values():
+    evs.SetPrimaryMetrics(evs.primary_metrics | {metric_name})
+
+  wmt24_tasks, wts = tasks.WMT24(wmt24_lps, k=0)
+  results = wmt24_tasks.Run(eval_set_dict=evs_dict)
+  metrics = {"average_correlation": results.AverageCorrs(wts)[metric_name]}
+
+  if args.output_file:
+    dirname = os.path.dirname(args.output_file)
+    if dirname:
+      os.makedirs(dirname, exist_ok=True)
+    with open(args.output_file, "w") as out:
+      out.write(json.dumps(metrics, indent=2))
+
+
+if __name__ == "__main__":
+  main()

vecalign/metricx24/models.py

@@ -0,0 +1,198 @@
+# coding=utf-8
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model classes for MetricX, modified from the T5 versions in HF."""
+
+import copy
+import dataclasses
+from typing import Optional, Tuple, Union
+import warnings
+
+import torch
+from torch import nn
+import transformers
+import transformers.modeling_outputs
+
+BaseModelOutput = transformers.modeling_outputs.BaseModelOutput
+ModelOutput = transformers.modeling_outputs.ModelOutput
+
+MT5Config = transformers.models.mt5.modeling_mt5.MT5Config
+MT5PreTrainedModel = transformers.models.mt5.modeling_mt5.MT5PreTrainedModel
+MT5Stack = transformers.models.mt5.modeling_mt5.MT5Stack
+
+__HEAD_MASK_WARNING_MSG = (
+    transformers.models.mt5.modeling_mt5.__HEAD_MASK_WARNING_MSG  # pylint: disable=protected-access
+)
+
+
+@dataclasses.dataclass
+class MT5ForRegressionOutput(ModelOutput):
+  loss: Optional[torch.FloatTensor] = None
+  predictions: torch.FloatTensor = None
+
+
+class MT5ForRegression(MT5PreTrainedModel):
+  """MT5 model for regression."""
+
+  def __init__(self, config: MT5Config):
+    super().__init__(config)
+    self.model_dim = config.d_model
+
+    self.shared = nn.Embedding(config.vocab_size, config.d_model)
+
+    encoder_config = copy.deepcopy(config)
+    encoder_config.is_decoder = False
+    encoder_config.use_cache = False
+    encoder_config.is_encoder_decoder = False
+    self.encoder = MT5Stack(encoder_config, self.shared)
+
+    decoder_config = copy.deepcopy(config)
+    decoder_config.is_decoder = True
+    decoder_config.is_encoder_decoder = False
+    decoder_config.num_layers = config.num_decoder_layers
+    self.decoder = MT5Stack(decoder_config, self.shared)
+
+    self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+    # Initialize weights and apply final processing
+    self.post_init()
+
+    # Model parallel
+    self.model_parallel = False
+    self.device_map = None
+
+  def forward(
+      self,
+      input_ids: Optional[torch.LongTensor] = None,
+      attention_mask: Optional[torch.FloatTensor] = None,
+      decoder_attention_mask: Optional[torch.BoolTensor] = None,
+      head_mask: Optional[torch.FloatTensor] = None,
+      decoder_head_mask: Optional[torch.FloatTensor] = None,
+      cross_attn_head_mask: Optional[torch.Tensor] = None,
+      encoder_outputs: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+      past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+      inputs_embeds: Optional[torch.FloatTensor] = None,
+      decoder_inputs_embeds: Optional[torch.FloatTensor] = None,
+      labels: Optional[torch.FloatTensor] = None,
+      use_cache: Optional[bool] = None,
+      output_attentions: Optional[bool] = None,
+      output_hidden_states: Optional[bool] = None,
+      return_dict: Optional[bool] = None,
+  ) -> Union[Tuple[torch.FloatTensor], MT5ForRegressionOutput]:
+    use_cache = use_cache if use_cache is not None else self.config.use_cache
+    return_dict = (
+        return_dict if return_dict is not None else self.config.use_return_dict
+    )
+
+    # FutureWarning: head_mask was separated into two input args - head_mask,
+    # decoder_head_mask
+    if head_mask is not None and decoder_head_mask is None:
+      if self.config.num_layers == self.config.num_decoder_layers:
+        warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning)
+        decoder_head_mask = head_mask
+
+    # Encode if needed (training, first prediction pass)
+    if encoder_outputs is None:
+      # Convert encoder inputs in embeddings if needed
+      encoder_outputs = self.encoder(
+          input_ids=input_ids,
+          attention_mask=attention_mask,
+          inputs_embeds=inputs_embeds,
+          head_mask=head_mask,
+          output_attentions=output_attentions,
+          output_hidden_states=output_hidden_states,
+          return_dict=return_dict,
+      )
+    elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
+      encoder_outputs = BaseModelOutput(
+          last_hidden_state=encoder_outputs[0],
+          hidden_states=encoder_outputs[1]
+          if len(encoder_outputs) > 1
+          else None,
+          attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
+      )
+
+    hidden_states = encoder_outputs[0]
+
+    if self.model_parallel:
+      torch.cuda.set_device(self.decoder.first_device)
+
+    # Create 1 step of dummy input for the decoder.
+    batch_size = input_ids.size(0)
+    decoder_input_ids = torch.LongTensor([0]).repeat(batch_size).reshape(-1, 1)
+    if torch.cuda.is_available():
+      decoder_input_ids = decoder_input_ids.to(torch.device("cuda"))
+
+    # Set device for model parallelism
+    if self.model_parallel:
+      torch.cuda.set_device(self.decoder.first_device)
+      hidden_states = hidden_states.to(self.decoder.first_device)
+      if decoder_input_ids is not None:
+        decoder_input_ids = decoder_input_ids.to(self.decoder.first_device)
+      if attention_mask is not None:
+        attention_mask = attention_mask.to(self.decoder.first_device)
+      if decoder_attention_mask is not None:
+        decoder_attention_mask = decoder_attention_mask.to(
+            self.decoder.first_device
+        )
+
+    # Decode
+    decoder_outputs = self.decoder(
+        input_ids=decoder_input_ids,
+        attention_mask=decoder_attention_mask,
+        inputs_embeds=decoder_inputs_embeds,
+        past_key_values=past_key_values,
+        encoder_hidden_states=hidden_states,
+        encoder_attention_mask=attention_mask,
+        head_mask=decoder_head_mask,
+        cross_attn_head_mask=cross_attn_head_mask,
+        use_cache=use_cache,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+    )
+
+    sequence_output = decoder_outputs[0]
+
+    # Set device for model parallelism
+    if self.model_parallel:
+      torch.cuda.set_device(self.encoder.first_device)
+      self.lm_head = self.lm_head.to(self.encoder.first_device)
+      sequence_output = sequence_output.to(self.lm_head.weight.device)
+
+    if self.config.tie_word_embeddings:
+      # Rescale output before projecting on vocab
+      # See
+      # https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586
+      sequence_output = sequence_output * (self.model_dim**-0.5)
+
+    lm_logits = self.lm_head(sequence_output)
+
+    # 250089 = <extra_id_10>
+    predictions = lm_logits[:, 0, 250089]
+
+    # Clip to 0 to 25
+    predictions = torch.clamp(predictions, 0, 25)
+
+    loss = None
+    if labels is not None:
+      loss_fct = nn.MSELoss()
+      # move labels to correct device to enable PP
+      labels = labels.to(predictions.device)
+      loss = loss_fct(predictions.view(-1), labels.view(-1))
+
+    return MT5ForRegressionOutput(
+        loss=loss,
+        predictions=predictions,
+    )

vecalign/metricx24/predict.py

@@ -0,0 +1,191 @@
+# coding=utf-8
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runs inference with a MetricX model."""
+
+import dataclasses
+import json
+import os
+
+import datasets
+# from metricx24 import models
+from . import models
+import torch
+import transformers
+import time
+
+
+@dataclasses.dataclass
+class Arguments:
+  """Prediction command-line arguments."""
+
+  tokenizer: str = dataclasses.field(
+      metadata={"help": "The name of the tokenizer"},
+  )
+
+  model_name_or_path: str = dataclasses.field(
+      metadata={
+          "help": (
+              "Path to pretrained model or model identifier from"
+              " huggingface.co/models"
+          )
+      },
+  )
+
+  max_input_length: int = dataclasses.field(
+      metadata={"help": "The maximum allowable input sequence length."},
+  )
+
+  batch_size: int = dataclasses.field(
+      metadata={"help": "The global prediction batch size."},
+  )
+
+  input_file: str = dataclasses.field(metadata={"help": "The input file."})
+
+  output_file: str = dataclasses.field(
+      metadata={"help": "The output file with predictions."},
+  )
+
+  qe: bool = dataclasses.field(
+      metadata={"help": "Indicates the metric is a QE metric."},
+      default=False,
+  )
+  device: str = dataclasses.field(
+      metadata={"help": "No device."},
+      default='0'
+  )
+
+
+def get_dataset(
+    input_file: str, tokenizer, max_input_length: int, device, is_qe: bool
+):
+  """Gets the test dataset for prediction.
+
+  If `is_qe` is true, the input data must have "hypothesis" and "source" fields.
+  If it is false, there must be "hypothesis" and "reference" fields.
+
+  Args:
+    input_file: The path to the jsonl input file.
+    tokenizer: The tokenizer to use.
+    max_input_length: The maximum input sequence length.
+    device: The ID of the device to put the PyTorch tensors on.
+    is_qe: Indicates whether the metric is a QE metric or not.
+
+  Returns:
+    The dataset.
+  """
+
+  def _make_input(example):
+    if is_qe:
+      example["input"] = (
+          "source: "
+          + example["source"]
+          + " candidate: "
+          + example["hypothesis"]
+      )
+    else:
+      example["input"] = (
+          "source: "
+          + example["source"]
+          + " candidate: "
+          + example["hypothesis"]
+          + " reference: "
+          + example["reference"]
+      )
+    return example
+
+  def _tokenize(example):
+    return tokenizer(
+        example["input"],
+        max_length=max_input_length,
+        truncation=True,
+        padding=False,
+    )
+
+  def _remove_eos(example):
+    example["input_ids"] = example["input_ids"][:-1]
+    example["attention_mask"] = example["attention_mask"][:-1]
+    return example
+
+  ds = datasets.load_dataset("json", data_files={"test": input_file})
+  ds = ds.map(_make_input)
+  ds = ds.map(_tokenize)
+  ds = ds.map(_remove_eos)
+  ds.set_format(
+      type="torch",
+      columns=["input_ids", "attention_mask"],
+      device=device,
+      output_all_columns=True,
+  )
+  return ds
+
+
+def main() -> None:
+  parser = transformers.HfArgumentParser(Arguments)
+  (args,) = parser.parse_args_into_dataclasses()
+  
+  os.environ['CUDA_VISIBLE_DEVICES']=args.device
+  os.environ['NCCL_P2P_DISABLE'] = "1"
+  os.environ['NCCL_IB_DISABLE'] = "1"
+
+  if torch.cuda.is_available():
+    device = torch.device(f"cuda:0")
+    per_device_batch_size = args.batch_size // torch.cuda.device_count()
+  else:
+    device = torch.device("cpu")
+    per_device_batch_size = args.batch_size
+
+  tokenizer = transformers.AutoTokenizer.from_pretrained(args.tokenizer)
+
+  model = models.MT5ForRegression.from_pretrained(
+      args.model_name_or_path, torch_dtype="auto"
+  )
+
+  model.to(device)
+  model.eval()
+
+  ds = get_dataset(
+      args.input_file,
+      tokenizer,
+      args.max_input_length,
+      device,
+      args.qe,
+  )
+
+  training_args = transformers.TrainingArguments(
+      output_dir=os.path.dirname(args.output_file),
+      per_device_eval_batch_size=per_device_batch_size,
+      dataloader_pin_memory=False,
+  )
+  trainer = transformers.Trainer(
+      model=model,
+      args=training_args,
+  )
+  predictions, _, _ = trainer.predict(test_dataset=ds["test"])
+
+  dirname = os.path.dirname(args.output_file)
+  if dirname:
+    os.makedirs(dirname, exist_ok=True)
+
+  with open(args.output_file, "w") as out:
+    for pred, example in zip(predictions, ds["test"]):
+      example["prediction"] = float(pred)
+      del example["input"]
+      del example["input_ids"]
+      del example["attention_mask"]
+      out.write(json.dumps(example, ensure_ascii=False) + "\n")
+
+
+if __name__ == "__main__":
+  main()

vecalign/overlap.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+import argparse
+
+from dp_utils import yield_overlaps
+
+
+def go(output_file, input_files, num_overlaps):
+    output = set()
+    for fin in input_files:
+        lines = open(fin, 'rt', encoding="utf-8").readlines()
+        for out_line in yield_overlaps(lines, num_overlaps):
+            output.add(out_line)
+
+    # for reproducibility
+    output = list(output)
+    output.sort()
+
+    with open(output_file, 'wt', encoding="utf-8") as fout:
+        for line in output:
+            fout.write(line + '\n')
+
+
+def _main():
+    parser = argparse.ArgumentParser('Create text file containing overlapping sentences.',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('-i', '--inputs', type=str, nargs='+',
+                        help='input text file(s).')
+
+    parser.add_argument('-o', '--output', type=str,
+                        help='output text file containing overlapping sentneces')
+
+    parser.add_argument('-n', '--num_overlaps', type=int, default=4,
+                        help='Maximum number of allowed overlaps.')
+
+    args = parser.parse_args()
+    go(output_file=args.output,
+       num_overlaps=args.num_overlaps,
+       input_files=args.inputs)
+
+
+if __name__ == '__main__':
+    _main()

vecalign/plan2align.py

@@ -0,0 +1,748 @@
+import openai
+from openai import OpenAI
+import spacy
+import pandas as pd
+from collections import defaultdict
+import random
+import torch
+import torch.nn as nn
+from transformers import MT5Tokenizer, MT5ForConditionalGeneration
+import shutil
+import os
+import subprocess
+import json
+from safetensors.torch import load_file
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from trl import AutoModelForCausalLMWithValueHead
+from huggingface_hub import login
+import logging
+import argparse
+
+lang_map = {
+    "English": ("en", "en_core_web_sm"),
+    "Russian": ("ru", "ru_core_news_sm"),
+    "German": ("de", "de_core_news_sm"),
+    "Japanese": ("ja", "ja_core_news_sm"),
+    "Korean": ("ko", "ko_core_news_sm"),
+    "Spanish": ("es", "es_core_news_sm"),
+    "Chinese": ("zh", "zh_core_web_sm")
+}
+
+openai = OpenAI(
+    api_key="",
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+MODEL_NAME= "google/gemma-2-9b-it" # "meta-llama/Meta-Llama-3.1-8B-Instruct"
+
+################################# folder / file processing #################################
+
+def clear_folder(folder_path):
+    if os.path.exists(folder_path):
+        shutil.rmtree(folder_path)
+        os.makedirs(folder_path)
+    else:
+        os.makedirs(folder_path)
+
+def delete_files_with_mt(folder_path):
+    if not os.path.exists(folder_path):
+        print(f"Folder {folder_path} does not exist.")
+        return
+    for filename in os.listdir(folder_path):
+        if "mt" in filename:
+            file_path = os.path.join(folder_path, filename)
+            try:
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                    print(f"Deleted file: {file_path}")
+            except Exception as e:
+                print(f"Failed to delete {file_path}. Reason: {e}")
+
+################################# reward model for ranking #################################
+
+class metricx_RewardModel:
+    def __init__(self):
+        self.device = "cuda:0"
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        self.json_path = os.path.join(current_dir, f'json_for_metricx')
+        if not os.path.exists(self.json_path):
+            os.makedirs(self.json_path)
+
+    def get_entry(self, src, mt):
+        return {"source": src, "hypothesis": mt, "reference": ""}
+
+    def write_jsonl(self, src_list, mts, session_id):
+        with open(os.path.join(self.json_path, f"{session_id}_input.jsonl"), 'w', encoding='utf-8') as output_file:
+            for src, mt in zip(src_list, mts):
+                entry = self.get_entry(src, mt)
+                output_file.write(json.dumps(entry, ensure_ascii=False) + '\n')
+                
+    def run_command(self, session_id):
+        devices_map = {'cuda:0':0, 'cuda:1':1, 'cuda:2':2, 'cuda:3':3}
+        command = [
+            "python", "-m", "vecalign.metricx24.predict",
+            "--tokenizer", "google/mt5-large",
+            "--model_name_or_path", "google/metricx-24-hybrid-large-v2p6",
+            "--max_input_length", "1536",
+            "--batch_size", "1",
+            "--input_file", os.path.join(self.json_path, f"{session_id}_input.jsonl"),
+            "--output_file", os.path.join(self.json_path, f"{session_id}_output.jsonl"),
+            "--device", f"{devices_map.get(self.device, 0)}",
+            "--qe"
+        ]
+        subprocess.run(command)
+
+    def get_predict(self, session_id):
+        scores = []
+        with open(os.path.join(self.json_path, f"{session_id}_output.jsonl"), 'r', encoding='utf-8') as new_file:
+            for line in new_file:
+                entry = json.loads(line)
+                score = entry.get('prediction', None)
+                scores.append(score)
+        clear_folder(self.json_path)
+        return scores
+
+    def reward_fn_batch(self, language, src_list, mts, session_id):
+        self.write_jsonl(src_list, mts, session_id)
+        self.run_command(session_id)
+        scores = self.get_predict(session_id)
+        rewards = [1 - (score / 25) for score in scores]
+        return rewards
+    
+reward_model = metricx_RewardModel()
+
+def batch_rm_find_best_translation(evals, language, session_id):
+    """
+    evals: list of (src, [translation1, translation2, ...])
+    Return the translation with the highest reward in each group that meets the THRESHOLD, along with its score.
+    Otherwise, return (None, score), where score is the highest score in that group.
+    """
+    src_list = []
+    mt_list = []
+    counts = []
+    for src, translations in evals:
+        counts.append(len(translations))
+        for mt in translations:
+            src_list.append(src)
+            mt_list.append(mt)
+    rewards = reward_model.reward_fn_batch(language, src_list, mt_list, session_id)
+    print("rewards: ", rewards)
+    best_translations = []
+    index = 0
+    for (src, translations), count in zip(evals, counts):
+        group_rewards = rewards[index: index+count]
+        index += count
+        if count < 2:
+            if translations:
+                best_translations.append((translations[0], group_rewards[0]))
+            else:
+                best_translations.append((None, None))
+        else:
+            best_index = group_rewards.index(max(group_rewards))
+            best_score = group_rewards[best_index]
+            if best_score >= THRESHOLD:
+                best_translations.append((translations[best_index], best_score))
+            else:
+                best_translations.append((None, best_score))
+    return best_translations
+
+
+def external_find_best_translation(evals, language, session_id):
+    """
+    evals: list of (src, [translation1, translation2, ...])
+    Return the translation with the highest reward in each group that meets the THRESHOLD, along with its score.
+    Otherwise, return (None, score), where score is the highest score in that group.
+    """
+    src_list = []
+    mt_list = []
+    counts = []
+    for src, translations in evals:
+        counts.append(len(translations))
+        for mt in translations:
+            src_list.append(src)
+            mt_list.append(mt)
+    rewards = reward_model.reward_fn_batch(language, src_list, mt_list, session_id)
+    print("rewards: ", rewards)
+    best_translations = []
+    index = 0
+    for (src, translations), count in zip(evals, counts):
+        group_rewards = rewards[index: index+count]
+        index += count
+        if count < 2:
+            if translations:
+                best_translations.append((translations[0], group_rewards[0]))
+            else:
+                best_translations.append((None, None))
+        else:
+            best_index = group_rewards.index(max(group_rewards))
+            best_score = group_rewards[best_index]
+            best_translations.append((translations[best_index], best_score))
+    return best_translations
+
+################################# generating translation #################################
+
+def translate_with_deepinfra(source_sentence, buffer, good_sent_size, src_language, tgt_language):    
+    system_prompts = [
+        "You are a meticulous translator. Provide a literal, word-for-word translation that preserves the structure and meaning of each individual word.",
+        "You are a professional translator. Deliver a clear, formal, and precise translation that faithfully conveys the original meaning.",
+        "You are a creative and expressive translator. Render the text in a vivid and imaginative way, as if narrating a captivating story."
+    ]
+    
+    context_prompt =  f"Below is a specialized, intermediate translation task. The input text is a mix of {src_language} and partial {tgt_language} translations. "
+    context_prompt += f"In the text, some {src_language} sentences are already followed by preliminary {tgt_language} translations enclosed in parentheses. "
+    context_prompt += f"These provided translations are rough references – they may be incomplete, inconsistent, or not fully aligned with the original meaning.\n\n"
+    context_prompt += f"Your task is to produce an improved {tgt_language} translation according to the following guidelines:\n"
+    context_prompt += f"1. **Refinement:** For sections with existing {tgt_language} translations (in parentheses), refine and polish them so that they are fluent, accurate, and coherent, fully capturing the meaning of the corresponding {src_language} text.\n"
+    context_prompt += f"2. **Completion:** For sections that remain untranslated, translate the {src_language} text accurately and naturally in the specified style.\n"
+    context_prompt += f"3. **Translation Order and Structure Preservation:** Maintain the original order and structure of the text. Every {src_language} sentence must appear in the same sequence as in the source text, with its corresponding {tgt_language} translation (if available) inserted immediately after it. Do not rearrange or reorder any part of the text.\n"
+    context_prompt += f"4. **Consistency:** Ensure a uniform tone and style across the entire translation, adhering to the translator role specified.\n"
+    context_prompt += f"5. **Final Output:** Provide the final output as a single, well-structured {tgt_language} text. Do not include any extraneous commentary, explanations, annotations, or headers – output only the translation in the correct order.\n\n"
+    context_prompt += f"Note: This translation is an intermediate version that may later be merged with other translations. Focus on clarity, coherence, and fidelity to the source text.\n"
+
+    # Process the buffer to extract relevant English translations
+    processed_source = source_sentence
+    if len(buffer) > 0:
+        selected_keys = random.sample(buffer.keys(), min(len(buffer), good_sent_size))
+        for key_sentence in selected_keys:
+            key_sentence = key_sentence.strip()
+            if key_sentence and (key_sentence in source_sentence) :
+                translated_sentence =  buffer[key_sentence][0][0]          
+                if f"\n({translated_sentence})\n" not in processed_source:
+                    processed_source = processed_source.replace(
+                        key_sentence, 
+                        f"{key_sentence}\n({translated_sentence})\n"
+                    )
+
+    context_prompt += f"\nHere is the input data for translation:\n{processed_source}\n\n"
+    context_prompt += "Apply the above guidelines to produce an improved, coherent translation that strictly follows the original order of the text.\n"
+    
+    if len(buffer) == 0:
+        context_prompt = f"### Translate this from {src_language} to {tgt_language} and only output the result."
+        context_prompt += f"\n### {src_language}:\n {source_sentence}"
+        context_prompt += f"\n### {tgt_language}:\n"
+
+    print("--------------------------------------------------------------------------------")
+    print("\n context_prompt \n")
+    print(context_prompt)
+    print("--------------------------------------------------------------------------------")
+    
+    translations = []
+    for prompt in system_prompts:
+        response = openai.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[
+                {"role": "system", "content": prompt},
+                {"role": "user", "content": context_prompt}
+            ]
+        )
+        translation = response.choices[0].message.content.strip()
+
+        print("--------------------------------------------------------------------------------")
+        print("\n rollout translation: \n")
+        print(translation)
+        print("--------------------------------------------------------------------------------")
+
+        translations.append(translation)
+    
+    return translations
+
+def process_buffer_sentences(source_sentences, buffer):
+    translations = []    
+    translation_map = {}
+    for src_key, trans_list in buffer.items():
+        if not trans_list or not isinstance(trans_list, list):
+            continue
+        src_sentences = [src_key]
+        
+        if len(src_sentences) > 0:
+            for src_sent in src_sentences:
+                if src_sent not in translation_map:
+                    translation_map[src_sent] = []
+                translation_map[src_sent] = trans_list[0]
+    
+    for src_sent in source_sentences:
+        if src_sent in translation_map and translation_map[src_sent]:
+            translations.append(translation_map[src_sent][0])
+    return translations
+
+def final_translate_with_deepinfra(source_sentence, source_segments, buffer, src_language, tgt_language):
+    translations = process_buffer_sentences(source_segments, buffer)
+    initial_translation = "\n".join(translations)
+
+    rewrite_prompt = (
+        f"Below is an initial translation of a {src_language} text into {tgt_language}. "
+        f"This translation may include omissions, inaccuracies, or awkward phrasing. "
+        f"Your task is to produce a refined version that is fluent, accurate, and coherent, "
+        f"while faithfully preserving the full meaning of the original {src_language} text.\n\n"
+        f"### Instructions:\n"
+        f"1. Ensure that every detail in the original {src_language} text is accurately represented.\n"
+        f"2. Correct any grammatical errors, unnatural expressions, or inconsistencies.\n"
+        f"3. Improve the natural flow so that the translation reads as if written by a native speaker.\n"
+        f"4. Do not add, omit, or change any essential details from the source text.\n"
+        f"5. Output only the final refined translation without any additional commentary.\n\n"
+        f"### Original {src_language} Text:\n{source_sentence}\n\n"
+        f"### Initial {tgt_language} Translation:\n{initial_translation}\n\n"
+        f"### Refined Translation:"
+    )
+
+    print("rewrite prompt:")
+    print(rewrite_prompt)
+
+    rewrite_response = openai.chat.completions.create(
+        model=MODEL_NAME,  # Replace with your actual model name
+        messages=[
+            {"role": "system", "content": "You are a helpful translator and only output the result."},
+            {"role": "user", "content": rewrite_prompt}
+        ]
+    )
+    translation = rewrite_response.choices[0].message.content.strip()
+    return translation
+
+
+################################# alignment functions #################################
+
+
+def save_sentences_to_txt(sentences, filename):
+    i = 0
+    with open(filename, "w", encoding="utf-8") as file:
+        for sentence in sentences:
+            print(sentence, i)
+            file.write(sentence + "\n")
+            i += 1
+
+def segment_sentences_by_punctuation(text, lang):
+    segmented_sentences = []
+    paragraphs = text.split('\n')
+    for paragraph in paragraphs:
+        if paragraph.strip():
+            if lang == src_lang:
+                doc = src_nlp(paragraph)
+            if lang == tgt_lang:
+                doc = mt_nlp(paragraph)
+            for sent in doc.sents:
+                segmented_sentences.append(sent.text.strip())
+    return segmented_sentences
+
+def generate_overlap_and_embedding(txt_file):
+    overlaps_file = txt_file + ".overlaps"
+    embed_file = txt_file + ".emb"
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    overlap_path = os.path.join(current_dir, "overlap.py")  
+    subprocess.run([overlap_path, "-i", txt_file, "-o", overlaps_file, "-n", "10"])
+    embed_command = [
+        "$LASER/tasks/embed/embed.sh",
+        overlaps_file,
+        embed_file,
+    ]
+    subprocess.run(" ".join(embed_command), shell=True)
+    return overlaps_file, embed_file
+
+def run_vecalign(src_txt, tgt_txt, src_embed, tgt_embed):
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    vecalign_path = os.path.join(current_dir, "vecalign.py")
+    result = subprocess.run(
+        [
+            "python", 
+            vecalign_path,
+            "--alignment_max_size", "8",
+            "--src", src_txt,
+            "--tgt", tgt_txt,
+            "--src_embed", src_txt + ".overlaps", src_embed,
+            "--tgt_embed", tgt_txt + ".overlaps", tgt_embed,
+        ],
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    alignments = []
+    for line in result.stdout.strip().split("\n"):
+        if line:
+            src_indices, tgt_indices, _ = line.split(":")
+            src_indices = list(map(int, src_indices.strip("[]").split(","))) if src_indices.strip("[]") else []
+            tgt_indices = list(map(int, tgt_indices.strip("[]").split(","))) if tgt_indices.strip("[]") else []
+            alignments.append((src_indices, tgt_indices))
+    return alignments
+
+def compute_alignment_stats(alignment_results):
+    costs = []
+    zero_cost_count = 0
+
+    for entry in alignment_results:
+        try:
+            cost = float(entry.split(":")[-1])  # Extract the cost value
+            if cost == 0.0:
+                zero_cost_count += 1
+            else:
+                costs.append(cost)
+        except ValueError:
+            continue  # Ignore invalid entries
+    
+    # Compute the average cost, ignoring zero-cost samples
+    avg_cost = sum(costs) / len(costs) if costs else 0.0
+    zero_cost_ratio = zero_cost_count / len(alignment_results) if alignment_results else 0.0
+
+    return avg_cost, zero_cost_ratio
+
+def run_vecalign_explore(src_txt, tgt_txt, src_embed, tgt_embed):
+    """
+    Runs vecalign multiple times, exploring the best del_percentile_frac.
+    Starts from 0.2 and decreases in 0.005 steps, stopping when zero-cost ratio increases sharply.
+    
+    :param src_txt: Source text file
+    :param tgt_txt: Target text file
+    :param src_embed: Source embeddings file
+    :param tgt_embed: Target embeddings file
+    :return: (best_del_percentile_frac, best_avg_cost, best_zero_cost_ratio, best_alignments)
+    """
+    del_percentile_frac = 0.2  # Starting value
+    step_size = 0.005  # Exploration step
+    prev_zero_cost_ratio = None
+    prev_avg_cost = None
+
+    best_avg_cost = float('inf')
+    best_del_percentile_frac = del_percentile_frac
+    best_zero_cost_ratio = 0.0
+    best_alignments = []
+
+    first_flag = True
+    first_zero_cost_ratio = 0.0
+    
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    vecalign_path = os.path.join(current_dir, "vecalign.py")
+
+    while del_percentile_frac > 0:
+        result = subprocess.run(
+            [
+                "python", 
+                vecalign_path,
+                "--alignment_max_size", "8",
+                "--del_percentile_frac", str(del_percentile_frac),
+                "--src", src_txt,
+                "--tgt", tgt_txt,
+                "--costs_sample_size", "200000", 
+                "--search_buffer_size", "20",
+                "--src_embed", src_txt + ".overlaps", src_embed,
+                "--tgt_embed", tgt_txt + ".overlaps", tgt_embed,
+            ],
+            stdout=subprocess.PIPE,
+            text=True,
+        )
+
+        output_lines = result.stdout.strip().split("\n")
+        avg_cost, zero_cost_ratio = compute_alignment_stats(output_lines)
+
+        print(f"del_percentile_frac: {del_percentile_frac:.3f} | Avg Cost: {avg_cost:.6f} | Zero-Cost Ratio: {zero_cost_ratio:.6%}")
+
+        if first_flag:
+            first_zero_cost_ratio = zero_cost_ratio
+            first_flag = False        
+
+        if prev_zero_cost_ratio != 0 and prev_zero_cost_ratio is not None and (zero_cost_ratio / prev_zero_cost_ratio) > 1.5:
+            print(f"Stopping exploration: Zero-cost ratio increased sharply at {del_percentile_frac:.3f}")
+            break
+        elif prev_zero_cost_ratio is not None and (
+            (zero_cost_ratio - prev_zero_cost_ratio) > 0.15  or
+            avg_cost > prev_avg_cost or
+            avg_cost < 0.3 or zero_cost_ratio > 0.7
+        ):
+            print(f"Stopping exploration: Zero-cost ratio increased sharply at {del_percentile_frac:.3f}")
+            break
+        else:
+            if avg_cost < best_avg_cost:
+                best_avg_cost = avg_cost
+                best_del_percentile_frac = del_percentile_frac
+                best_zero_cost_ratio = zero_cost_ratio
+                best_alignments = output_lines
+        
+        prev_zero_cost_ratio = zero_cost_ratio
+        prev_avg_cost = avg_cost
+        del_percentile_frac -= step_size
+
+    final_avg_cost = best_avg_cost
+    final_zero_cost_ratio = best_zero_cost_ratio 
+    final_del_percentile_frac = best_del_percentile_frac
+    final_alignments = best_alignments.copy()
+
+    parsed_alignments = []
+    for line in final_alignments:
+        if line:
+            src_indices, tgt_indices, _ = line.split(":")
+            src_indices = list(map(int, src_indices.strip("[]").split(","))) if src_indices.strip("[]") else []
+            tgt_indices = list(map(int, tgt_indices.strip("[]").split(","))) if tgt_indices.strip("[]") else []
+            parsed_alignments.append((src_indices, tgt_indices))
+
+    print("\nBest Found:")
+    print(f"del_percentile_frac: {final_del_percentile_frac:.3f} | Avg Cost: {final_avg_cost:.6f} | Zero-Cost Ratio: {final_zero_cost_ratio:.6%}")
+
+    return parsed_alignments
+
+def standardize_common_alignments(common_alignments_list):
+    # Reference alignment for standardization (use the shortest alignment set as baseline)
+    reference_alignments = min(common_alignments_list, key=lambda alignments: len(alignments))
+
+    # Standardized results to return
+    standardized_results = []
+
+    for alignments in common_alignments_list:
+        standardized_alignment = []
+        mt_idx_map = {tuple(src): mt for src, mt in alignments}
+        for src_indices, _ in reference_alignments:  # Ignore ref_indices as it no longer exists
+            # If src_indices exist in the current alignment, use them directly
+            if tuple(src_indices) in mt_idx_map:
+                mt_indices = mt_idx_map[tuple(src_indices)]
+            else:
+                # If not found, merge based on src alignment
+                mt_indices = []
+                for src in src_indices:
+                    if (src,) in mt_idx_map:
+                        mt_indices.extend(mt_idx_map[(src,)])
+                # Ensure indices are unique and sorted after merging
+                mt_indices = sorted(set(mt_indices))
+            standardized_alignment.append((src_indices, mt_indices))
+        standardized_results.append(standardized_alignment)
+    return standardized_results
+
+def generate_windows(source, translations):
+    # Segment sentences
+    source_segments = segment_sentences_by_punctuation(source, lang=src_lang)   
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    temp_folder = os.path.join(current_dir, "temp")
+    os.makedirs(temp_folder, exist_ok=True)
+    # Generate overlaps and embeddings
+    src_txt = os.path.join(current_dir, f"temp/{SESSION_ID}_src.txt")
+    mt_txt = os.path.join(current_dir, f"temp/{SESSION_ID}_mt.txt")
+
+    print("\n ----------------- source segmentation --------------------------- ")
+    save_sentences_to_txt(source_segments, src_txt)
+    print(" -------------------------------------------------------------------  \n")
+    _, src_embed = generate_overlap_and_embedding(src_txt)
+    mt_segments_list = [segment_sentences_by_punctuation(t, lang=tgt_lang) for t in translations]
+    adjusted_mt_list = []
+    
+    common_alignments_list = []
+    for mt_segments in mt_segments_list:
+        print("\n ----------------- translation segmentation --------------------------- ")
+        save_sentences_to_txt(mt_segments, mt_txt)
+        print(" ------------------------------------------------------------------------  \n")
+        _, mt_embed = generate_overlap_and_embedding(mt_txt)
+        src_mt_alignments = run_vecalign_explore(src_txt, mt_txt, src_embed, mt_embed) # run_vecalign_explore, run_vecalign
+        common_alignments_list.append(src_mt_alignments.copy())
+        delete_files_with_mt(temp_folder)
+    
+    common_alignments_list = standardize_common_alignments(common_alignments_list)
+
+    mt_index = 0
+   
+    for common_alignments in common_alignments_list:
+        adjusted_src = []
+        adjusted_mt = []
+        for src_indices, mt_indices in common_alignments:
+            mt_indices = [x for x in mt_indices if x != -1]
+            
+            if len(src_indices) == 0:
+                continue
+            else:
+                aligned_src = " ".join([source_segments[i] for i in src_indices])
+            
+            if len(mt_indices) > 0:
+                aligned_mt = " ".join([mt_segments_list[mt_index][i] for i in mt_indices])
+            else:
+                aligned_mt = ""
+            
+            adjusted_src.append(aligned_src)
+            adjusted_mt.append(aligned_mt)
+
+        adjusted_mt_list.append(adjusted_mt.copy())
+        mt_index += 1
+    
+    clear_folder(temp_folder)
+    return adjusted_src, adjusted_mt_list
+
+################################# main function #################################
+
+def saving_memory(buffer, index, iteration, final_translations_record):
+    """
+    Save the buffer, and final_translations_record to the Memory folder.
+    """
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    memory_folder = os.path.join(current_dir, f"{MEMORY_FOLDER}")
+    os.makedirs(memory_folder, exist_ok=True)
+    buffer_file_path = f"{MEMORY_FOLDER}/buffer_{index}_iter_{iteration}.json"
+    metadata_file_path = f"{MEMORY_FOLDER}/metadata_{index}_iter_{iteration}.json"
+
+    buffer_to_save = {key: list(value) for key, value in buffer.items()}
+    with open(buffer_file_path, "w", encoding="utf-8") as f:
+        json.dump(buffer_to_save, f, ensure_ascii=False, indent=4)
+    
+    metadata = {
+        "final_translations_record": final_translations_record
+    }
+    with open(metadata_file_path, "w", encoding="utf-8") as f:
+        json.dump(metadata, f, ensure_ascii=False, indent=4)
+
+    print(f"Buffer saved to {buffer_file_path}")
+    print(f"Metadata saved to {metadata_file_path}")
+
+
+def process_chunk():
+
+    data = pd.read_csv(csv_path)
+    for index, row in data.iterrows():
+        print("::::::::::::::::::::::: index :::::::::::::::::::::::", index, " ::::::::::::::::::::::: index :::::::::::::::::::::::", )
+        buffer = defaultdict(list)
+                
+        source_sentence = row[src_lang].replace('\n', ' ')
+        source_segments = segment_sentences_by_punctuation(source_sentence, lang=src_lang)
+
+        for iteration in range(max_iterations):
+            print(f"\nStarting iteration {iteration + 1}/{max_iterations}...\n")
+            
+            if iteration in stop_memory:
+                final_translations = final_translate_with_deepinfra(source_sentence, source_segments, buffer, SRC_LANGUAGE, TASK_LANGUAGE)
+                print("Final Translation Method:")
+                print(final_translations)
+                final_translations_record = [final_translations]
+                saving_memory(buffer, index, iteration, final_translations_record)
+            
+            if iteration == max_iterations - 1:
+                break
+            else:
+                translations = translate_with_deepinfra(source_sentence, buffer, good_ref_contexts_num+iteration, SRC_LANGUAGE, TASK_LANGUAGE)
+            
+            src_windows, mt_windows_list = generate_windows(source_sentence, translations)
+
+            ####################################### Evaluate translations and update buffer #######################################
+            print("Evaluate translations and update buffer ..............")
+
+            # First, store all sources and candidate translations as lists.
+            src_context_list = list(src_windows)
+            candidates_list = []
+            for window_index in range(len(src_windows)):
+                candidates = [mt_windows[window_index] for mt_windows in mt_windows_list]
+                candidates_list.append(candidates)
+            
+            # Batch evaluate all candidate translations, returning the best translation and score for each source.
+            best_candidate_results = batch_rm_find_best_translation(list(zip(src_context_list, candidates_list)), TASK_LANGUAGE)
+
+            print("\n Our best candidate results:")
+            print(best_candidate_results)
+            print(" ------------------------------------------------------------------------  \n")
+
+            print("\n===== Initial buffer state =====")
+            for src, translations in buffer.items():
+                print(f"Source '{src}': {[t[0] for t in translations]}")
+
+            # Update the buffer for each source.
+            for i, src in enumerate(src_context_list):
+                best_tuple = best_candidate_results[i]  # (translation, score)
+                if best_tuple[0] is not None:
+                    # If the source is not yet in the buffer, initialize it.
+                    if src not in buffer:
+                        buffer[src] = [best_tuple]
+                        print(f"[ADD] New Source '{src}' Add Translation: '{best_tuple[0]}', Score: {best_tuple[1]}")
+                    else:
+                        # Directly add the new translation to the buffer.
+                        buffer[src].append(best_tuple)
+                        print(f"[ADD] Source '{src}' Add Translation: '{best_tuple[0]}', Score: {best_tuple[1]}")
+                    
+                    # Sort by score to place the best translation (highest score) at the top.
+                    buffer[src].sort(key=lambda x: x[1], reverse=True)
+                    print(f"[UPDATE] Source '{src}' Best Translation: '{buffer[src][0][0]}'")
+
+            print("\n===== Final buffer state =====")
+            for src, translations in buffer.items():
+                print(f"Source '{src}': {[t[0] for t in translations]}")
+
+
+        print("Final Translation:")
+        print(final_translations)
+
+
+def get_lang_and_nlp(language):
+    if language not in lang_map:
+        raise ValueError(f"Unsupported language: {language}")
+    lang_code, model_name = lang_map[language]
+    return lang_code, spacy.load(model_name)
+
+def translate_text(text, session_id, 
+                            src_language="Japanese", 
+                            task_language="English", 
+                            max_iterations_value=3, 
+                            threshold_value=0.7, 
+                            good_ref_contexts_num_value=5, 
+                            reward_model_type='metricx'):
+
+    global SRC_LANGUAGE, TASK_LANGUAGE, max_iterations, stop_memory
+    global THRESHOLD, good_ref_contexts_num, src_lang, src_nlp, tgt_lang, mt_nlp
+    global reward_model, MEMORY_FOLDER, SESSION_ID
+    
+    SESSION_ID = session_id
+    print("SESSION_ID: ", SESSION_ID)
+    
+    MEMORY_FOLDER = "external_translation_memory"
+    SRC_LANGUAGE = src_language
+    TASK_LANGUAGE = task_language
+    max_iterations = max_iterations_value
+    stop_memory = list(range(1, max_iterations))
+    THRESHOLD = threshold_value
+    good_ref_contexts_num = good_ref_contexts_num_value
+
+    import torch
+    device = torch.device(f"cuda:0" if torch.cuda.is_available() else "cpu")
+    src_lang, src_nlp = get_lang_and_nlp(SRC_LANGUAGE)
+    tgt_lang, mt_nlp = get_lang_and_nlp(TASK_LANGUAGE)
+
+    reward_model = metricx_RewardModel()
+    
+    from collections import defaultdict
+    buffer = defaultdict(list)
+    source_sentence = text.replace("\n", " ")
+    source_segments = segment_sentences_by_punctuation(source_sentence, lang=src_lang)
+    final_translations = None
+
+    for iteration in range(max_iterations):
+        # print(f"\nStarting iteration {iteration + 1}/{max_iterations}...\n")
+        if iteration in stop_memory:
+            final_translations = final_translate_with_deepinfra(source_sentence, source_segments, buffer, SRC_LANGUAGE, TASK_LANGUAGE)
+            # saving_memory(buffer, 0, iteration, [final_translations])
+        if iteration == max_iterations - 1:
+            break
+        else:
+            translations = translate_with_deepinfra(source_sentence, buffer, good_ref_contexts_num + iteration, SRC_LANGUAGE, TASK_LANGUAGE)
+        
+        src_windows, mt_windows_list = generate_windows(source_sentence, translations)
+        # print("Evaluate translations and update buffer ..............")
+
+        src_context_list = list(src_windows)
+        candidates_list = []
+        for window_index in range(len(src_windows)):
+            candidates = [mt_windows[window_index] for mt_windows in mt_windows_list]
+            candidates_list.append(candidates)
+
+        best_candidate_results = batch_rm_find_best_translation(list(zip(src_context_list, candidates_list)), TASK_LANGUAGE, SESSION_ID)
+
+        # print("\n Best candidate results:")
+        # print(best_candidate_results)
+        # print(" ------------------------------------------------------------------------\n")
+
+        for i, src in enumerate(src_context_list):
+            best_tuple = best_candidate_results[i]
+            if best_tuple[0] is not None:
+                if src not in buffer:
+                    buffer[src] = [best_tuple]
+                    # print(f"[ADD] New Source '{src}' Add Translation: '{best_tuple[0]}', Score: {best_tuple[1]}")
+                else:
+                    buffer[src].append(best_tuple)
+                    # print(f"[ADD] Source '{src}' Add Translation: '{best_tuple[0]}', Score: {best_tuple[1]}")
+                buffer[src].sort(key=lambda x: x[1], reverse=True)
+                # print(f"[UPDATE] Source '{src}' Best Translation: '{buffer[src][0][0]}'")
+
+        # print("\n===== Buffer state =====")
+        for src, translations in buffer.items():
+            print(f"Source '{src}': {[t[0] for t in translations]}")
+    
+    # print("Final Translation:")
+    # print(final_translations)
+    return final_translations
+
+
+if __name__ == "__main__":
+    process_chunk()

vecalign/score.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import argparse
+import sys
+from collections import defaultdict
+
+import numpy as np
+
+from dp_utils import read_alignments
+
+"""
+Faster implementation of lax and strict precision and recall, based on
+   https://www.aclweb.org/anthology/W11-4624/.
+
+"""
+
+
+def _precision(goldalign, testalign):
+    """
+    Computes tpstrict, fpstrict, tplax, fplax for gold/test alignments
+    """
+    tpstrict = 0  # true positive strict counter
+    tplax = 0     # true positive lax counter
+    fpstrict = 0  # false positive strict counter
+    fplax = 0     # false positive lax counter
+
+    # convert to sets, remove alignments empty on both sides
+    testalign = set([(tuple(x), tuple(y)) for x, y in testalign if len(x) or len(y)])
+    goldalign = set([(tuple(x), tuple(y)) for x, y in goldalign if len(x) or len(y)])
+
+    # mappings from source test sentence idxs to
+    #    target gold sentence idxs for which the source test sentence 
+    #    was found in corresponding source gold alignment
+    src_id_to_gold_tgt_ids = defaultdict(set)
+    for gold_src, gold_tgt in goldalign:
+        for gold_src_id in gold_src:
+            for gold_tgt_id in gold_tgt:
+                src_id_to_gold_tgt_ids[gold_src_id].add(gold_tgt_id)
+
+    for (test_src, test_target) in testalign:
+        if (test_src, test_target) == ((), ()):
+            continue
+        if (test_src, test_target) in goldalign:
+            # strict match
+            tpstrict += 1
+            tplax += 1
+        else:
+            # For anything with partial gold/test overlap on the source,
+            #   see if there is also partial overlap on the gold/test target
+            # If so, its a lax match
+            target_ids = set()
+            for src_test_id in test_src:
+                for tgt_id in src_id_to_gold_tgt_ids[src_test_id]:
+                    target_ids.add(tgt_id)
+            if set(test_target).intersection(target_ids):
+                fpstrict += 1
+                tplax += 1
+            else:
+                fpstrict += 1
+                fplax += 1
+
+    return np.array([tpstrict, fpstrict, tplax, fplax], dtype=np.int32)
+
+
+def score_multiple(gold_list, test_list, value_for_div_by_0=0.0):
+    # accumulate counts for all gold/test files
+    pcounts = np.array([0, 0, 0, 0], dtype=np.int32)
+    rcounts = np.array([0, 0, 0, 0], dtype=np.int32)
+    for goldalign, testalign in zip(gold_list, test_list):
+        pcounts += _precision(goldalign=goldalign, testalign=testalign)
+        # recall is precision with no insertion/deletion and swap args
+        test_no_del = [(x, y) for x, y in testalign if len(x) and len(y)]
+        gold_no_del = [(x, y) for x, y in goldalign if len(x) and len(y)]
+        rcounts += _precision(goldalign=test_no_del, testalign=gold_no_del)
+
+    # Compute results
+    # pcounts: tpstrict,fnstrict,tplax,fnlax
+    # rcounts: tpstrict,fpstrict,tplax,fplax
+
+    if pcounts[0] + pcounts[1] == 0:
+        pstrict = value_for_div_by_0
+    else:
+        pstrict = pcounts[0] / float(pcounts[0] + pcounts[1])
+
+    if pcounts[2] + pcounts[3] == 0:
+        plax = value_for_div_by_0
+    else:
+        plax = pcounts[2] / float(pcounts[2] + pcounts[3])
+
+    if rcounts[0] + rcounts[1] == 0:
+        rstrict = value_for_div_by_0
+    else:
+        rstrict = rcounts[0] / float(rcounts[0] + rcounts[1])
+
+    if rcounts[2] + rcounts[3] == 0:
+        rlax = value_for_div_by_0
+    else:
+        rlax = rcounts[2] / float(rcounts[2] + rcounts[3])
+
+    if (pstrict + rstrict) == 0:
+        fstrict = value_for_div_by_0
+    else:
+        fstrict = 2 * (pstrict * rstrict) / (pstrict + rstrict)
+
+    if (plax + rlax) == 0:
+        flax = value_for_div_by_0
+    else:
+        flax = 2 * (plax * rlax) / (plax + rlax)
+
+    result = dict(recall_strict=rstrict,
+                  recall_lax=rlax,
+                  precision_strict=pstrict,
+                  precision_lax=plax,
+                  f1_strict=fstrict,
+                  f1_lax=flax)
+
+    return result
+
+
+def log_final_scores(res):
+    print(' ---------------------------------', file=sys.stderr)
+    print('|             |  Strict |    Lax  |', file=sys.stderr)
+    print('| Precision   |   {precision_strict:.3f} |   {precision_lax:.3f} |'.format(**res), file=sys.stderr)
+    print('| Recall      |   {recall_strict:.3f} |   {recall_lax:.3f} |'.format(**res), file=sys.stderr)
+    print('| F1          |   {f1_strict:.3f} |   {f1_lax:.3f} |'.format(**res), file=sys.stderr)
+    print(' ---------------------------------', file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        'Compute strict/lax precision and recall for one or more pairs of gold/test alignments',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('-t', '--test', type=str, nargs='+', required=True,
+                        help='one or more test alignment files')
+
+    parser.add_argument('-g', '--gold', type=str, nargs='+', required=True,
+                        help='one or more gold alignment files')
+
+    args = parser.parse_args()
+
+    if len(args.test) != len(args.gold):
+        raise Exception('number of gold/test files must be the same')
+
+    gold_list = [read_alignments(x) for x in args.gold]
+    test_list = [read_alignments(x) for x in args.test]
+
+    res = score_multiple(gold_list=gold_list, test_list=test_list)
+    log_final_scores(res)
+
+
+if __name__ == '__main__':
+    main()

vecalign/standalone_document_embedding_demo.py

@@ -0,0 +1,100 @@
+
+#!/usr/bin/env python3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This is a standalone example of creating a document vector from sentence vectors
+following https://aclanthology.org/2020.emnlp-main.483 
+
+"""
+
+
+import numpy as np
+
+from mcerp import PERT  # pip install mcerp  # see https://github.com/tisimst/mcerp/blob/master/mcerp/__init__.py
+
+
+NUM_TIME_SLOTS = 16
+PERT_G = 20
+
+
+# PERT is very slow (50ms per distribution) so we cache a bank of PERT distributions
+_num_banks = 100
+_xx = np.linspace(start=0, stop=1, num=NUM_TIME_SLOTS)
+PERT_BANKS = []
+for _pp in np.linspace(0, 1, num=_num_banks):
+    if _pp == 0.5:  # some special case that makes g do nothing
+        _pp += 0.001
+    pert = PERT(low=-0.001, peak=_pp, high=1.001, g=PERT_G, tag=None)
+    _yy = pert.rv.pdf(_xx)
+    _yy = _yy / sum(_yy)  # normalize
+    PERT_BANKS.append(_yy)
+
+
+np.set_printoptions(threshold=50, precision=5)
+    
+
+def build_doc_embedding(sent_vecs, sent_counts):
+    # ensure sentence counds are >= 1
+    sent_counts = np.clip(sent_counts, a_min=1, a_max=None)
+
+    # scale each sent vec by 1/count
+    sent_weights = 1.0/np.array(sent_counts)
+
+    scaled_sent_vecs = np.multiply(sent_vecs.T, sent_weights).T
+
+    # equally space sentences
+    sent_centers = np.linspace(0, 1, len(scaled_sent_vecs))
+
+    # find weighting for each sentence, for each time slot
+    sentence_loc_weights = np.zeros((len(sent_centers), NUM_TIME_SLOTS))
+
+    for sent_ii, p in enumerate(sent_centers):
+        bank_idx = int(p * (len(PERT_BANKS) - 1))  # find the nearest cached pert distribution
+        sentence_loc_weights[sent_ii, :] = PERT_BANKS[bank_idx]
+        
+    # make each chunk vector
+    doc_chunk_vec = np.matmul(scaled_sent_vecs.T, sentence_loc_weights).T
+
+    # concatenate chunk vectors into a single vector for the full document
+    doc_vec = doc_chunk_vec.flatten()
+
+    # normalize document vector
+    doc_vec = doc_vec / (np.linalg.norm(doc_vec) + 1e-5)
+
+    return doc_vec
+
+
+def demo():
+
+    # Replace sent_vecs with laser/LaBSE/etc embeddings of each sentence in your document,
+    #    after projecting the sentence embeddings into a lower-dimensional space using something like PCA (see paper for details).
+    sent_emb_size = 32  # Document embedding size will be sent_emb_size * NUM_TIME_SLOTS
+    n_sents = 7
+    sent_vecs = np.random.rand(n_sents, sent_emb_size)-0.5
+    
+    # Replace sent_counts with the number of times each sentence has been seen in your corpus.
+    sent_counts = np.random.randint(low=1, high=50, size=n_sents)
+
+    doc_emb = build_doc_embedding(sent_vecs, sent_counts)
+    
+    print('Document Embedding:', doc_emb)
+    print('Document Embedding Size:', doc_emb.shape)
+
+
+if __name__ == '__main__':
+    demo()

vecalign/valid_en_ja.csv

@@ -0,0 +1,322 @@
+en_url,en_title,en,jp_url,jp_title,ja
+https://developer.nvidia.com/blog/expanding-ai-agent-interface-options-with-2d-and-3d-digital-human-avatars/,Expanding AI Agent Interface Options with 2D and 3D Digital Human Avatars,"When interfacing with
+generative AI
+applications, users have multiple communication options—text, voice, or through digital avatars.
+Traditional chatbot or copilot applications have text interfaces where users type in queries and receive text-based responses. For hands-free communication, speech AI technologies like
+automatic speech recognition
+(ASR) and
+text-to-speech
+(TTS) facilitate verbal interactions, ideal for scenarios like phone-based customer service. Moreover, combining digital avatars with speech capabilities provides a more dynamic interface for users to engage visually with the application. According to Gartner, by 2028, 45% of organizations with more than 500 employees will leverage employee AI avatars to expand the capacity of human capital.
+1
+Digital avatars can vary widely in style—some use cases benefit from photorealistic 3D or 2D avatars, while other use cases work better with a stylized, or cartoonish avatar.
+3D Avatars
+offer fully immersive experiences, showcasing lifelike movements and photorealism. Developing these avatars requires specialized software and technical expertise, as they involve intricate body animations and high-quality renderings.
+2D Avatars
+are quicker to develop and ideal for web-embedded solutions. They offer a streamlined approach to creating interactive AI, often requiring artists for design and animation but less intensive in terms of technical resources.
+To kickstart your creation of a photo-realistic digital human, the
+NVIDIA AI Blueprint on digital humans for customer service
+can be tailored for various use cases. This functionality is now included with support for the NVIDIA Maxine
+Audio2Face-2D
+NIM microservice. ‌Additionally, the blueprint now offers flexibility in rendering for 3D avatar developers to use
+Unreal Engine
+.
+How to add a talking digital avatar to your agent application
+In the AI Blueprint for digital humans, a user interacts with an
+AI agent
+that leverages
+NVIDIA ACE
+technology (Figure 1).
+Figure 1. Architecture diagram for the NVIDIA AI Blueprint for digital humans
+The audio input from the user is sent to the ACE agent which orchestrates the communication between various NIM microservices. The ACE agent uses the
+Riva Parakeet NIM
+to convert the audio to text, which is then processed by a RAG pipeline. The RAG pipeline uses the NVIDIA NeMo Retriever
+embedding
+and
+reranking
+NIM microservices, and an
+LLM NIM
+, to respond with relevant context from stored documents.
+Finally, the response is converted back to speech via Riva TTS, animating the digital human using the Audio2Face-3D NIM or Audio2Face-2D NIM.
+Considerations when designing your AI agent application
+In global enterprises, communication barriers across languages can slow down operations. AI-powered avatars with multilingual capabilities communicate across languages effortlessly. The digital human AI Blueprint provides conversational AI capabilities that simulate human interactions that accommodate users’ speech styles and languages through Riva ASR, neural machine translation (NMT) along with intelligent interruption and barge-in support.
+One of the key benefits of digital human AI agents is their ability to function as “always-on” resources for employees and customers alike. RAG-powered AI agents continuously learn from interactions and improve over time, providing more accurate responses and better user experiences.
+For enterprises considering digital human interfaces, choosing the right avatar and rendering option depends on the use case and customization preferences.
+Use Case
+: 3D avatars are ideal for highly immersive use cases like in physical stores, kiosks or primarily one-to-one interactions, while 2D avatars are effective for web or mobile conversational AI use cases.
+Development and Customization Preferences
+: Teams with 3D and animation expertise can leverage their skillset to create an immersive and ultra-realistic avatar, while teams looking to iterate and customize quickly can benefit from the simplicity of 2D avatars.
+Scaling Considerations:
+Scaling is an important consideration when evaluating avatars and corresponding rendering options. Stream throughput, especially for 3D avatars, is highly dependent on the choice and quality of the character asset used, the desired output resolution and the rendering option of choice (Omniverse Renderer or Unreal Engine) can play a critical role in determining per stream compute footprint.
+NVIDIA Audio2Face-2D allows creation of lifelike 2D avatars from just a portrait image and voice input. Easy and simple configurations allow developers to quickly iterate and produce target avatars and animations for their digital human use cases. With real-time output and cloud-native deployment, 2D digital humans are ideal for interactive use cases and streaming avatars for interactive web-embedded solutions.
+For example, enterprises looking to deploy AI agents across multiple devices and inserting digital humans into web- or mobile-first customer journeys, can benefit from the reduced hardware demands of 2D avatars.
+3D photorealistic avatars provide an unmatched immersive experience for use cases demanding ‌highly empathetic user engagement. NVIDIA Audio2Face-3D and Animation NIM microservices animate a 3D character by generating blendshapes along with subtle head and body animation to create an immersive, photorealistic avatar. The digital human AI Blueprint now supports two rendering options for 3D avatars, including Omniverse Renderer and Unreal Engine Renderer, providing developers the flexibility to integrate the rendering option of their choice.
+To explore how digital humans can enhance your enterprise, visit the
+NVIDIA API catalog
+to learn about the different avatar options.
+Getting started with digital avatars
+For hands-on development with Audio2Face-2D and Unreal Engine NIM microservices,
+apply for ACE Early Access
+or dive into the digital human AI Blueprint
+technical blog
+to learn how you can add digital human interfaces to personalize chatbot applications.
+1
+Gartner®, Hype Cycle for the Future of Work, 2024 by Tori Paulman, Emily Rose McRae, etc., July 2024
+GARTNER is a registered trademark and service mark of Gartner, Inc. and/or its affiliates in the U.S. and internationally and is used herein with permission. All rights reserved.",https://developer.nvidia.com/ja-jp/blog/expanding-ai-agent-interface-options-with-2d-and-3d-digital-human-avatars/,2D と 3D のデジタル ヒューマン アバターによる AI エージェント インターフェイス オプションの拡張,"Reading Time:
+2
+minutes
+ユーザーが
+生成 AI
+アプリケーションを使ってやり取りする際には、テキスト、音声、デジタル アバターなど複数のコミュニケーション オプションを利用することができます。
+従来のチャットボットやコパイロット アプリケーションでは、ユーザーが問い合わせを入力し、テキストベースの応答を受信するテキスト インターフェイスを使用しています。ハンズフリーのコミュニケーションでは、
+自動音声認識
+(ASR: Automatic Speech Recognition) や
+音声合成
+(TTS: Text-To-Speech) などの音声 AI 技術により、電話を使用したカスタマー サービスなどのシナリオに最適な口頭によるやり取りが容易になります。さらに、デジタル アバターに音声機能を持たせることで、ユーザーがアプリケーションを視覚的に使用できるため、ダイナミックなインターフェイスを提供できます。Gartner によると、2028 年までに、従業員 500 名以上の組織の 45% が、人的資本の能力拡大のために、 AI アバターの従業員を活用するようになるそうです。
+1
+デジタル アバターのスタイルは様々で、フォトリアリスティックな 3D または 2D のアバターが適しているケースもあれば、定型化されたアバターや漫画のようなアバターの方が適しているケースもあります。
+3D アバター
+は、リアルな動きと写実性を再現し、完全な没入体験を提供します。このようなアバターの開発には、複雑なボディー アニメーションや高品質のレンダリングが必要となるため、専門的なソフトウェアや技術的な専門知識が必要になります。
+2D アバター
+は開発が迅速で、Web に組み込みソリューションに最適です。インタラクティブな AI の作成に合理的なアプローチを提供し、デザインやアニメーションにはアーティストが必要になることが多いですが、技術的なリソースの面はそれほど負担になりません。
+フォトリアリスティックなデジタル ヒューマンの作成を始めるにあたり、
+カスタマー サービス向けデジタル ヒューマンの NVIDIA AI Blueprint
+は、さまざまなユース ケースに合わせてカスタマイズすることができます。この機能は現在、NVIDIA Maxine
+Audio2Face-2D
+NIM マイクロサービスのサポートに含まれています。さらに、この Blueprint では、3D アバター開発者が
+Unreal Engine
+を使用できるよう、レンダリングに柔軟性を持たせています。
+エージェント アプリケーションに会話するデジタル アバターを追加する方法
+デジタル ヒューマン向け AI Blueprint では、ユーザーが
+NVIDIA ACE
+技術を活用した
+AI エージェント
+と対話します (図 1)。
+図 1. デジタル ヒューマン向け NVIDIA AI Blueprint のアーキテクチャ
+ユーザーによる音声入力は、さまざまな NIM マイクロサービス間の通信を調整する ACE エージェントに送��されます。ACE エージェントは、
+Riva Parakeet NIM
+を使用して音声をテキストに変換し、そのテキストは RAG パイプラインで処理されます。RAG パイプラインでは、NIM マイクロサービスの
+埋め込み
+と
+リランク
+を行う NVIDIA NeMo Retriever と
+LLM NIM
+を使用して、保存されたドキュメントから関連するコンテキストを用いて応答します。
+最後に、Riva TTS を介してこの応答を音声に変換し、Audio2Face-3D NIM または Audio2Face-2D NIM を使用してデジタル ヒューマンをアニメーション化します。
+AI エージェント アプリケーションを設計する際に考慮すべきポイント
+グローバル企業では、言語の壁によるコミュニケーションの障害が業務の妨げとなることがあります。多言語機能を備えた AI 搭載アバターを使用すれば、言語の壁を超えた円滑なコミュニケーションを取ることができます。デジタル ヒューマン AI Blueprint は、Riva ASR やニューラル機械翻訳 (NMT: Neural Machine Translation) に加え、インテリジェントな割り込みやバージイン機能を備え、ユーザーの話し方や言語に柔軟に対応できる、人間らしい対話型 AI を実現します。
+デジタル ヒューマン AI エージェントの主な利点の 1 つは、従業員と顧客の両者にとって「常時稼働する」リソースとして機能できることです。RAG を搭載した AI エージェントは、やりとりから継続的に学習し、時間の経過とともに改善していくため、より正確な対応とより優れたユーザー体験を提供することができます。
+デジタル ヒューマン インターフェイスを検討している企業にとって、適切なアバターとレンダリング オプションの選択は、ユース ケースやカスタマイズ設定に依存します。
+ユース ケース
+: 3D アバターは、実店舗やキオスク (無人端末) など、主に 1対 1 のやりとりのような、非常に没入感の高いユース ケースに最適ですが、2D アバターは、Web やモバイルの対話型 AI ユース ケースに効果的です。
+開発とカスタマイズの設定
+: 3D やアニメーションの専門知識を持つチームは、そのスキルを活用して没入感のある超リアルなアバターを作成できます。一方、反復作業やカスタマイズを迅速に行いたいチームには、シンプルな 2D アバターが有効です。
+スケーリングの考慮すべきポイント
+: アバターと対応するレンダリング オプションを評価する際に、スケーリングは考慮すべき重要なポイントです。ストリームのスループットは、特に 3D アバターの場合、使用するキャラクター アセットの選択と品質によって大きく異なります。希望する出力解像度や選択するレンダリング オプション (Omniverse Renderer または Unreal Engine) は、ストリームあたりの計算フットプリントを決定する上で重要な役割を果たします。
+NVIDIA Audio2Face-2D では、顔写真と音声入力だけでリアルな 2D アバターを作成できます。簡単でシンプルな構成のため、開発者はデジタル ヒューマンのユース ケースに合わせたアバターやアニメーションを迅速に繰り返し作成できます。リアルタイム出力とクラウド ネイティブのデプロイにより、2D デジタル ヒューマンは、インタラクティブなユース ケースや、インタラクティブな Web 組み込みソリューション向けのストリーミング アバターに最適です。
+たとえば、複数のデバイスに AI エージェントをデプロイし、Web またはモバイル ファーストのカスタマー ジャーニーにデジタル ヒューマンを導入しようとしている企業には、2D アバターはハードウェア要件が軽減するのでメリットがあります。
+3D のフォトリアリスティックなアバターは、高い共感が要求されるユーザー エンゲージメントを必要とするユース ケースに、比類のない没入体験を提供します。NVIDIA Audio2Face-3D とアニメーション NIM マイクロサービスは、繊細な頭部と身体のアニメーションとともにブレンドシェイプを生成し、没入感のあるフォトリアリスティックなアバターを作成することで、3D キャラクターをアニメーション化します。デジタル ヒューマン AI Blueprint は、3D アバターのレンダリング オプションをとして、Omniverse レンダラーと Unreal-Engine レンダラーをサポートしており、開発者が選択したレンダリング オプションを柔軟に統合できるようになりました。
+デジタル ヒューマンが企業を強化する方法については、
+NVIDIA API カタログ
+にアクセスして、さまざまなアバターのオプションをご覧ください。
+デジタル アバターを始める
+Audio2Face-2D と Unreal Engine NIM マイクロサービスを使用した実践的な開発については、
+ACE 早期アクセスに申し込む
+か、デジタル ヒューマン AI Blueprint の
+技術ブログ
+にアクセスして、チャットボット アプリケーションをパーソナライズするためにデジタル ヒューマン インターフェイスを追加する方法について学ぶことができます。
+1
+Gartner®, Hype Cycle for the Future of Work, 2024 by Tori Paulman, Emily Rose McRae, etc., July 2024
+GARTNER is a registered trademark and service mark of Gartner, Inc. and/or its affiliates in the U.S. and internationally and is used herein with permission. All rights reserved.
+関連情報
+GTC セッション:
+Enhancing the Digital Human Experience with Cloud Microservices Accelerated by Generative AI
+GTC セッション:
+Build a World of Interactive Avatars Based on NVIDIA Omniverse, AIGC, and LLM
+NGC コンテナー:
+ACE エージェント サンプル フロントエンド
+SDK:
+NVIDIA Tokkio
+ウェビナー:
+How Telcos Transform Customer Experiences with Conversational AI"
+https://developer.nvidia.com/blog/5x-faster-time-to-first-token-with-nvidia-tensorrt-llm-kv-cache-early-reuse/,5x Faster Time to First Token with NVIDIA TensorRT-LLM KV Cache Early Reuse,"In our previous
+blog post
+, we demonstrated how reusing the key-value (KV) cache by offloading it to CPU memory can accelerate time to first token (TTFT) by up to 14x on x86-based NVIDIA H100 Tensor Core GPUs and 28x on the NVIDIA GH200 Superchip. In this post, we shed light on KV cache reuse techniques and best practices that can drive even further TTFT speedups.
+Introduction to KV cache
+LLM models are rapidly being adopted for many tasks, including question-answering, and code generation. To generate a response, these models begin by converting the user’s prompt into tokens, which are then transformed into dense vectors. Extensive dot-product operations follow to mathematically model the relationships between the tokens and build a contextual understanding of the user input. The computational cost of generating this contextual understanding increases quadratically with the length of the input sequence.
+This resource-intensive process generates keys and values, which are cached to avoid recomputation when generating subsequent tokens. Reusing the KV cache reduces the computational load and time needed to generate additional tokens—leading to a faster and more efficient user experience.
+When reusing the KV cache, careful attention must be given to how long it remains in memory, which components to evict first when memory is full, and when it can be reused for new incoming prompts. Optimizing these factors can lead to incremental performance improvements in KV cache reuse. NVIDIA TensorRT-LLM offers three key features that specifically address these areas.
+Early KV cache reuse
+Traditional reuse algorithms require the entire KV cache computation to be completed before any portions of it can be reused with new user prompts. In scenarios such as enterprise chatbots, where system prompts—predefined instructions added to user queries—are essential to direct the LLM’s responses in line with enterprise guidelines, this method can be inefficient.
+When a surge of users interacts with the chatbot simultaneously, each user would require a separate computation of the system prompt KV cache. With TensorRT-LLM, we can instead reuse the system prompt as it is being generated in real time, enabling it to be shared across all users during the burst, rather than recalculating it for each user. This can significantly accelerate inference for use cases requiring system prompts by up to 5x.
+Figure 1. TensorRT-LLM KV cache reuse can speed up TTFT by up to 5x
+Flexible KV cache block sizing
+In reuse implementations, only entire cache memory blocks can be allocated for reuse. For example, if the cache memory block size is 64 tokens and KV cache is 80 tokens, only 64 tokens will be stored for reuse, while the remaining 16 tokens will need to be recomputed. However, if the memory block size is reduced to 16 tokens, all 64 tokens can be stored across five memory blocks, eliminating the need for re-computation.
+This effect is most pronounced when the input sequences are short. For long input sequences, larger blocks can be more beneficial.  As is clear, the more granular the control you have over the KV cache, the better you can optimize it for your specific use case.
+TensorRT-LLM provides fine-grained control over KV cache memory blocks, giving developers the ability to chop them into smaller blocks between 64 to 2 tokens. This optimizes the usage of allocated memory, increases reuse rates, and improves TTFT. When running LLAMA70B on NVIDIA H100 Tensor Core GPUs, we can speed up TTFT up to 7% in multi-user environments by reducing KV cache block size from 64 tokens to 8 tokens.
+Figure 2. Impact of changing KV cache block size on inference speedup
+Efficient KV cache eviction protocols
+Partitioning the KV cache into smaller blocks and evicting unused ones can be effective for memory optimization, but it introduces dependency complexities. When a specific block is used to generate a response, and the result is stored as a new block, it can form a tree-like structure of dependencies.
+Over time, the counters tracking the usage of the source blocks (the branches) may become stale as the dependent nodes (the leaves) are reused. Evicting the source block then requires the eviction of all dependent blocks, which would require recalculation of the KV cache for new user prompts, increasing TTFT.
+To address this challenge, TensorRT-LLM includes intelligent eviction algorithms that can trace the dependent nodes from their source nodes and evict dependent nodes first, even if they have more recent reuse counters. This ensures more efficient memory management while preventing unnecessary evictions of dependent blocks.
+Figure 3. A logical representation of KV cache eviction algorithm show how it can reduce the number of evicted blocks, increasing the likelihood of reuse
+Getting started with TensorRT-LLM KV cache reuse
+Generating KV cache during inference requires a lot of compute and memory resources. Using it efficiently is critical to improving model response, accelerating inference, and increasing system throughput. TensorRT-LLM provides advanced reuse features for developers looking to further optimize TTFT response times for peak performance.
+To start using TensorRT-LLM KV cache reuse check out our
+GitHub documentation
+.",https://developer.nvidia.com/ja-jp/blog/5x-faster-time-to-first-token-with-nvidia-tensorrt-llm-kv-cache-early-reuse/,NVIDIA TensorRT-LLM の KV Cache Early Reuseで、Time to First Token を 5 倍高速化,"Reading Time:
+2
+minutes
+以前の
+ブログ記事
+では、key-value (KV) キャッシュを CPU メモリにオフロードして再利用することで、最初のトークンが出力されるまでの時間 (TTFT: Time To First Token) を x86 ベースの NVIDIA H100 Tensor コア GPU で最大 14 倍、NVIDIA GH200 Superchip で最大 28 倍に高速化できる方法をご紹介しました。本記事では、KV キャッシュの再利用技術と、TTFT のさらなる高速化を実現するベストプラクティスについて解説します。
+KV キャッシュの概要
+LLM モデルは、質問回答やコード生成など、多くのタスクで急速に採用されています。応答を生成するにあたり、これらのモデルはまず、ユーザーのプロンプトをトークンへ変換し、その後これらのトークンを密ベクトルへと変換します。膨大なドット積演算がその後に続き、その後トークン間の関係性を数学的にモデル化し、ユーザー入力に対する文脈理解を構築します。この文脈理解を生成するためにかかる計算コストは、入力シーケンスの長さの二乗に比例して増加します。
+このリソースを大量に消費するプロセスから key とvalue が生成され、後続のトークンを生成するときに再度計算されないようにキャッシュされます。KV キャッシュを再利用することで、追加のトークンを生成する際に必要となる計算負荷と時間が軽減され、より高速で効率的なユーザー体験を実現します。
+KV キャッシュを再利用するときには、キャッシュがメモリに残る期間、メモリが一杯になったときに最初に削除するコンポーネント、および新しい入力プロンプトに再利用できるタイミングなどの点に細心の注意を払う必要があります。これらの要因を最適化することで、KV キャッシュの再利用におけるパフォーマンスの段階的な増加へとつなげることができます。NVIDIA TensorRT-LLM は、これらの分野に特化した 3 つの主要な機能を提供します。
+Early KV cache reuse
+従来の再利用アルゴリズムでは、KV キャッシュをその一部であっても新しいユーザー プロンプトで再利用するためには、事前にすべての KV キャッシュの計算を完了させておく必要がありました。この方法は、LLM のレスポンスを企業のガイドラインに沿ったものにするために、システム プロンプト (ユーザーの問い合わせに追加される事前定義の指示) が不可欠となる企業向けチャットボットなどのシナリオでは、非効率的である可能性があります。
+チャットボットと同時にやり取りするユーザーが急増した場合、各ユーザーに対してシステム プロンプト KV キャッシュを個別に計算する必要があります。TensorRT-LLM では、リアルタイムで生成���れるシステム プロンプトを再利用することができるため、急増時にはすべてのユーザーと共有することができ、ユーザーごとに再計算する必要がありません。これにより、システム プロンプトを必要とするユース ケースの推論を最大 5 倍にまで高速化することができます。
+図 1. TensorRT-LLM KV cache reuse により、TTFT を最大 5 倍高速化
+柔軟な KV キャッシュ ブロック サイズ
+再利用を実装する際には、キャッシュ メモリ ブロック全体のみを再利用に割り当てることができます。例えば、キャッシュ メモリ ブロック サイズが 64 トークンで、KV キャッシュが 80 トークンである場合、再利用のために保存できるのは 64 トークンのみであり、残りの 16 トークンは再計算する必要があります。しかしながら、メモリ ブロック サイズを 16 トークンに減らすと、64 トークンすべてを 5 つのメモリ ブロックに格納することができ、再計算の必要性がなくなります。
+この効果は、入力シーケンスが短いときに最も顕著に現れます。長い入力シーケンスの場合は、より大きなブロックの方がより有益です。明らかに、KV キャッシュをより細かく制御できればできるほど、特定のユース ケースに合わせた最適化も向上します。
+TensorRT-LLM では、KV キャッシュ メモリ ブロックをきめ細かく制御できるため、開発者は KV キャッシュ メモリ ブロックを 64 から 2 トークンまで、より小さなブロックに分割することができます。これにより、割り当てられたメモリの使用が最適化され、再利用率が上昇し、TTFT が改善されます。NVIDIA H100 Tensor コア GPU で LLAMA70B を実行する場合、KV キャッシュ ブロックサイズを 64 トークンから 8 トークンへと減らすことで、マルチユーザー環境で TTFT を最大 7% 高速化できます。
+図 2. KV キャッシュ ブロック サイズの変更による推論の高速化
+効率的な KV キャッシュの除外 (Eviction) プロトコル
+KV キャッシュをより小さなブロックに分割し、未使用のブロックを除外することは、メモリの最適化に効果的ですが、依存関係に複雑さが生まれます。特定のブロックがレスポンスの生成に使用され、その結果が新しいブロックとして保存されると、依存関係のツリー構造が形成される可能性があります。
+時間の経過とともに、ソース ブロック (ブランチ) の使用を追跡するカウンターは、従属ノード (リーフ) が再利用されるにつれて古くなる可能性があります。ソース ブロックを除外するには、従属するすべてのブロックを除外する必要があり、新しいユーザ プロンプトの KV キャッシュを再計算する必要が生じて TTFT が増加します。
+この課題に対処するために、TensorRT-LLM には、従属ノードをソース ノードから追跡し、従属ノードがより最近の再利用カウンターを持っている場合でも、最初に従属ノードを除外することができるインテリジェントな除外アルゴリズムが含まれています。これにより、より効率的にメモリを管理できるようになると共に、従属ブロックの不要な除外を回避できます。
+図 3. KV キャッシュの除外アルゴリズムの論理を表現した図。除外されるブロックの数を減らし、再利用の可能性を高められる様子を示しています。
+TensorRT-LLM KV cache reuse を使い始める
+推論中に KV キャッシュを生成するには、多くの計算とメモリ ソースが必要になります。効率的に使用することが、モデル応答の改善、推論の高速化、システム スループットの向上には不可欠です。TensorRT-LLM は、ピーク性能のために TTFT 応答時間をさらに最適化しようとする開発者に高度な再利用機能を提供します。
+TensorRT-LLM KV cache reuse を使い始めるには、
+GitHub のドキュメント
+を参照してください。
+関連情報
+GTC セッション:
+Speeding up LLM Inference With TensorRT-LLM (TensorRT-LLM による LLM 推論の高速化)
+GTC セッション:
+Optimizing and Scaling LLMs With TensorRT-LLM for Text Generation (テキスト生成のための TensorRT-LLM を使用した LLM の最適化とスケーリング)
+SDK:
+Torch-TensorRT
+SDK:
+TensorRT
+SDK:
+TensorFlow-TensorRT"
+https://developer.nvidia.com/blog/state-of-the-art-multimodal-generative-ai-model-development-with-nvidia-nemo/,State-of-the-Art Multimodal Generative AI Model Development with NVIDIA NeMo,"Generative AI
+has rapidly evolved from text-based models to multimodal capabilities. These models perform tasks like image captioning and visual question answering, reflecting a shift toward more human-like AI. The community is now expanding from text and images to video, opening new possibilities across industries.
+Video AI models are poised to revolutionize industries such as robotics, automotive, and retail. In
+robotics
+, they enhance autonomous navigation in complex, ever-changing environments, which is vital for sectors like manufacturing and warehouse management. In the automotive industry, video AI is propelling autonomous driving, boosting vehicle perception, safety, and predictive maintenance to improve efficiency.
+To build image and video foundation models, developers must curate and preprocess a large amount of training data, tokenize the resulting high-quality data at high fidelity, train or customize pretrained models efficiently and at scale, and then generate high-quality images and videos during inference.
+Announcing NVIDIA NeMo for multimodal generative AI
+NVIDIA NeMo
+is an end-to-end platform for developing, customizing, and deploying generative AI models.
+NVIDIA just announced the expansion of NeMo to support the end-to-end pipeline for developing multimodal models. NeMo enables you to easily curate high-quality visual data, accelerate
+training
+and
+customization
+with highly efficient tokenizers and parallelism techniques, and reconstruct high-quality visuals during inference.
+Accelerated video and image data curation
+High-quality training data ensures high-accuracy results from an AI model. However, developers face various challenges in building data processing pipelines, ranging from scaling to data orchestration.
+NeMo Curator
+streamlines the data curation process, making it easier and faster for you to build multimodal generative AI models. Its out-of-the-box experience minimizes the total cost of ownership (TCO) and accelerates time-to-market.
+While working with visuals, organizations can easily reach petabyte-scale data processing. NeMo Curator provides an orchestration pipeline that can load balance on multiple GPUs at each stage of the data curation. As a result, you can reduce video processing time by 7x compared to a naive GPU-based implementation. The scalable pipelines can efficiently process over 100 PB of data, ensuring the seamless handling of large datasets.
+Figure 1. NVIDIA NeMo Curator video processing speed
+NeMo Curator provides reference video curation models optimized for high-throughput filtering, captioning, and embedding stages to enhance dataset quality, empowering you to create more accurate AI models.
+For instance, NeMo Curator uses an optimized captioning model that delivers an order of magnitude throughput improvement compared to unoptimized inference model implementations.
+NVIDIA Cosmos tokenizers
+Tokenizers map redundant and implicit visual data into compact and semantic tokens, enabling efficient training of large-scale generative models and democratizing their inference on limited computational resources.
+Today’s open video and image tokenizers often generate poor data representations, leading to lossy reconstructions, distorted images, and temporally unstable videos and placing a cap on the capability of generative models built on top of the tokenizers. Inefficient tokenization processes also result in slow encoding and decoding and longer training and inference times, negatively impacting both developer productivity and the user experience.
+NVIDIA Cosmos tokenizers are open models that offer superior visual tokenization with exceptionally large compression rates and cutting-edge reconstruction quality across diverse image and video categories.
+Video 1. Efficient Generative AI Tokenizers for Image and Video
+These tokenizers provide ease of use through a suite of tokenizer standardized models that support vision-language models (VLMs) with discrete latent codes, diffusion models with continuous latent embeddings, and various aspect ratios and resolutions, enabling the efficient management of large-resolution images and videos. This provides you with tools for tokenizing a wide variety of visual input data to build image and video AI models.
+Cosmos tokenizer architecture
+A Cosmos tokenizer uses a sophisticated encoder-decoder structure designed for high efficiency and effective learning. At its core, it employs 3D
+causal convolution blocks
+, which are specialized layers that jointly process spatiotemporal information, and uses causal temporal attention that captures long-range dependencies in data.
+The causal structure ensures that the model uses only past and present frames when performing tokenization, avoiding future frames. This is crucial for aligning with the causal nature of many real-world systems, such as those in physical AI or multimodal LLMs.
+Figure 2. NVIDIA Cosmos tokenizer architecture
+The input is downsampled using 3D wavelets, a signal processing technique that represents pixel information more efficiently. After the data is processed, an inverse wavelet transform reconstructs the original input.
+This approach improves learning efficiency, enabling the tokenizer encoder-decoder learnable modules to focus on meaningful features rather than redundant pixel details. The combination of such techniques and its unique training recipe makes the Cosmos tokenizers a cutting-edge architecture for efficient and powerful tokenization.
+During inference, the Cosmos tokenizers significantly reduce the cost of running the model by delivering up to 12x faster reconstruction compared to leading open-weight tokenizers (Figure 3).
+Figure 3. Quantitative comparison of reconstruction quality (left) and runtime performance (right) for video tokenizers
+The Cosmos tokenizers also produce high-fidelity images and videos while compressing more than other tokenizers, demonstrating an unprecedented quality-compression trade-off.
+Figure 4. Continuous tokenizer compression rate compared to reconstruction quality
+Figure 5. Discrete tokenizer compression rate compared to reconstruction quality
+Although the Cosmos tokenizer regenerates from highly compressed tokens, it is capable of creating high-quality images and videos due to an innovative neural network training technique and architecture.
+Figure 6. Reconstructed video frame for continuous video tokenizers
+Build Your Own Multimodal Models with NeMo
+The expansion of the NVIDIA NeMo platform with at-scale data processing using
+NeMo Curator
+and high-quality tokenization and visual reconstruction using the Cosmos tokenizer empowers you to build state-of-the-art multimodal, generative AI models.
+Join the waitlist
+and be notified when NeMo Curator is available. The tokenizer is available now on the
+/NVIDIA/cosmos-tokenizer
+GitHub repo and
+Hugging Face
+.",https://developer.nvidia.com/ja-jp/blog/state-of-the-art-multimodal-generative-ai-model-development-with-nvidia-nemo/,NVIDIA NeMo による最先端のマルチモーダル生成 AI モデル開発,"Reading Time:
+2
+minutes
+生成 AI
+は、テキストベースのモデルからマルチモーダル機能へと急速に進化しています。これらのモデルは、画像のキャプション作成や視覚的な質問回答などのタスクを実行し、より人間に近い AI へとシフトしていることを反映しています。このコミュニティは現在、テキストや画像から動画へと拡大しており、さまざまな業界で新たな可能性を切り開かれています。
+動画 AI モデルは、ロボティクス、自動車、小売などの業界に革命を起こそうとしています。
+ロボティクス
+では、製造業や倉庫管理などの分野に不可欠な、複雑で変化し続ける環境における自律的なナビゲーションを強化しています。自動車業界では、動画 AI が自動運転を推進し、車両の認識、安全性、予知保全を強化し、効率性を高めています。
+画像や動画の基盤モデルを構築するには、開発者は大量の学習データのキュレーションと事前処理を行い、結果として得られた高品質データを高い忠実度でトークン化し、学習済みモデルを効率的に大規模に学習またはカスタマイズして、推論中に高品質な画像や動画を生成する必要があります。
+マルチモーダル生成 AI 向けの NVIDIA NeMo を発表
+NVIDIA NeMo
+は、生成 AI モデルを開発、カスタマイズ、デプロイするエンドツーエンドのプラットフォームです。
+NVIDIA は、マルチモーダル モデル開発向けのエンドツーエンドのパイプラインをサポートする NeMo の拡張を発表しました。NeMo により、高品質な視覚データを簡単にキュレーションし、高効率なトークナイザーと並列処理技術で
+学習
+と
+カスタマイズ
+を加速し、推論中に高品質なビジュアルを再構築することができます。
+動画と画像データのキュレーションを加速
+高品質な学習データでは、AI モデルから高精度な結果が得られます。しかし、開発者は、データ処理パイプラインの構築において、スケーリングからデータのオーケストレーションまで、さまざまな課題に直面しています。
+NeMo Curator
+は、データ キュレーション プロセスを合理化することで、マルチモーダル生成 AI モデルをより簡単かつ迅速に構築することができます。すぐに試すことができるため、総保有コスト (TCO) を最小限に抑え、市場投入までの時間を短縮します。
+ビジュアルを扱う際には、組織はペタバイト規模のデータ処理を容易に実行できます。NeMo Curator は、データ キュレーションの各段階で複数の GPU に負荷分散できるオーケストレーション パイプラインを提供します。その結果、単純な GPU ベースの実装と比較して、動画処理時間を 7 分の 1 に短縮できます。スケール可能なパイプラインは、100 PB を超えるデータを効率的に処理でき、大規模なデータセットをシームレスに取り扱うことができます。
+図 1. NVIDIA NeMo Curator の動画処理速度
+NeMo Curator は、高いスループットのフィルタリング、キャプション作成、埋め込みの各段階に最適化されたリファレンス ビデオ キュレーション モデルを提供し、データセットの品質を向上させ、より正確な AI モデルの作成をサポートします。
+たとえば、NeMo Curator は、最適化されたキャプション モデルを使用し、最適化されていない推論モデルの実装と比較して、桁違いのスループットの向上を実現します。
+NVIDIA Cosmos トークナイザー
+トークナイザーは、冗長的で暗黙的な視覚データをコンパクトで意味のあるトークンにマッピングし、大規模な生成モデルの効率的な学習を実現し、誰もが限られた計算リソースで推論できるようにします。
+今日のオープンな動画や画像のトークナイザーは、データ表現が不十分なことが多いため、劣化の多い再構築、歪んだ画像、不連続な動画につながり、トークナイザー上に構築された生成モデルの能力に限界をもたらします。トークン化プロセスが非効率なため、エンコードやデコードに時間がかかり、学習や推論の時間が長くなり、開発者の生産性とユーザー体験の両方に悪影響を及ぼします。
+NVIDIA Cosmos トークナイザーは、優れた視覚トークン化を提供するオープンなモデルで、さまざまな画像や動画のカテゴリーで、高い圧縮率と最先端の再構築品質を実現します。
+離散的な潜在コードを備えた視覚言語モデル (VLM: Vision-language Model)、連続した潜在的埋め込みによる拡散モデル、さまざまなアスペクト比や解像度をサポートする一連のトークナイザー標準化モデルを使用して、これらのトークナイザーを簡単に使用でき、高解像度の画像や動画を効率的に管理することができます。これにより、画像や動画 AI モデルを構築するために、幅広い視覚入力データをトークン化するツールが提供されます。
+Cosmos トークナイザーのアーキテクチャ
+Cosmos トークナイザーは、高効率かつ効果的な学習向けに設計されており、高度なエンコーダー / デコーダー構造を使用しています。その中核には 3D
+Causal Convolution Block
+(因果畳み込みブロック) を採用しています。これは時空間情報を共同処理する特殊なレイヤーで、データの長期的な依存関係を捉える Causal Temporal Attention (因果的時間注意機構) を使用しています。
+この因果構造により、トークン化の実行時にモデルが過去と現在のフレームのみを使用し、未来のフレームは使用しません。これは、物理的なAIやマルチモーダルLLMなどの多くの現実世界のシステムの因果性に合わせるために重要です。
+図 2. NVIDIA Cosmos トークナイザーのアーキテクチャ
+入力は、ピクセル情報をより効率的に表す信号処理技術である 3D ウェーブレットを使用してダウンサンプリングされます。データ処理後、逆ウェーベレット変換によって元の入力が再構築されます。
+このアプローチにより、学習効率が向上し、トークナイザーのエンコーダー / デコーダーの学習可能なモジュールは、冗長なピクセルの詳細ではなく、意味のある特徴に焦点を当てることができます。このような技術と独自の学習レシピの組み合わせにより、Cosmos トークナイザーは、効率的かつ強力なトークン化を実現する最先端のアーキテクチャとなっています。
+推論の際、Cosmos トークナイザーは、主要なオープンウェイトのトークナイザーと比較して最大 12 倍高速な再構築を実現し、モデルの実行コストを大幅に削減しました (図 3)。
+図 3. Cosmos トークナイザーと主要なオープンウェイトのトークナイザーとの比較
+Cosmos トークナイザーは、他のトークナイザーよりも高い圧縮率を実現しながら、高い忠実度の画像や動画を生成し、前例のない品質と圧縮のトレードオフを実現しています。
+図 4. 連続トークナイザーの圧縮率と再構築品質の比較
+図 5. 離散トークナイザーの圧縮率と再構築品質の比較
+Cosmos トークナイザーは、高度に圧縮されたトークンから再生成されますが、革新的なニューラル ネットワークの学習技術とアーキテクチャにより、高品質な画像や動画を作成することができます。
+図 6. 連続動画トークナイザーで再構築された動画フレーム
+NeMo で独自のマルチモーダル モデルを構築
+NeMo Curator
+を使用した大規模なデータ処理と、Cosmos トークナイザーを使用した高品質なトークン化やビジュアル再構築を備えた、NVIDIA NeMo プラットフォームの拡張により、最先端のマルチモーダル生成 AI モデルを構築することができます。
+登録
+していただくと、NeMo Curator が利用可能になった際に通知を受け取ることができます。トークナイザーは、現在
+/NVIDIA/cosmos-tokenizer
+GitHub リポジトリおよび
+Hugging Face
+で利用することができます。
+関連情報
+GTC セッション:
+Large Language Model Fine-Tuning using Parameter Efficient Fine-Tuning (PEFT を使用した大規模言語モデルのファインチューニング)
+GTC セッション:
+Large Language Model Fine-Tuning using NVIDIA NeMo (NVIDIA NeMo を使用した大規模言語モデルのファインチューニング – Domino Data Lab 提供)
+SDK:
+NVIDIA NeMo カスタマイザー
+SDK:
+NeMo LLM サービス
+SDK:
+NeMo Megatron"

vecalign/valid_zh_en.csv

@@ -0,0 +1,466 @@
+id,seg,en,zh,zh_title,zh_url,llama3,gpt4o
+1,0,"The financial services industry is reaching an important milestone with AI, as organizations move beyond testing and experimentation to successful AI implementation, driving business results.
+NVIDIA’s fifth annual State of AI in Financial Services report shows how financial institutions have consolidated their AI efforts to focus on core applications, signaling a significant increase in AI capability and proficiency.
+AI Helps Drive Revenue and Save Costs Companies investing in AI are seeing tangible benefits, including increased revenue and cost savings.
+Nearly 70% of respondents report that AI has driven a revenue increase of 5% or more, with a dramatic rise in those seeing a 10-20% revenue boost. In addition, more than 60% of respondents say AI has helped reduce annual costs by 5% or more. Nearly a quarter of respondents are planning to use AI to create new business opportunities and revenue streams.
+The top generative AI use cases in terms of return on investment (ROI) are trading and portfolio optimization, which account for 25% of responses, followed by customer experience and engagement at 21%. These figures highlight the practical, measurable benefits of AI as it transforms key business areas and drives financial gains.
+Overcoming Barriers to AI Success Half of management respondents said they’ve deployed their first generative AI service or application, with an additional 28% planning to do so within the next six months. A 50% decline in the number of respondents reporting a lack of AI budget suggests increasing dedication to AI development and resource allocation.
+The challenges associated with early AI exploration are also diminishing. The survey revealed fewer companies reporting data issues and privacy concerns, as well as reduced concern over insufficient data for model training. These improvements reflect growing expertise and better data management practices within the industry.
+As financial services firms allocate budget and grow more savvy at data management, they can better position themselves to harness AI for enhanced operational efficiency, security and innovation across business functions.","金融服務業在使用人工智慧（AI）方面正邁入一個重要的里程碑，各大組織開始邁出測試與實驗的範疇，成功使用 AI 推動業務成果。
+NVIDIA 的第五份《金融服務業 AI 現況（State of AI in Financial Services）》年度調查報告顯示，金融機構已經整合自身在 AI 方面的各項作為，以專注在核心應用項目上，這標誌著 AI 能力與熟練程度大幅提升。
+AI 有助於增加營收與節省成本
+投資於 AI 的公司正在看到實質效益，包括增加營收和節省成本等。
+近七成的受訪者表示，AI 已經帶來 5% 或以上的營收成長，其中營收成長幅度達 10% 至 20% 的受訪者比例更是大幅增加。此外，超過六成的受訪者表示 AI 已協助減少 5% 或以上的年度成本。近四分之一的受訪者正計劃使用 AI 創造新的商機和收入來源。
+交易與投資組合最佳化是投資報酬率（ROI）最高的
+生成式 AI 使用案例，佔回應數量的 25%，其次是客戶體驗與參與度，佔 21%。這些數字突顯 AI在改變關鍵業務領域和推動財務收益時，所帶來可衡量的實際效益。
+克服 AI 成功的關卡
+半數管理層的受訪者表示，他們已經部署了第一個生成式 AI 服務或應用，另有 28% 的受訪者計劃在未來六個月內部署。回覆缺乏 AI 預算的受訪者人數減少了五成，這顯示對於 AI 開發與資源分配的投入程度日益增加。
+與早期探索 AI 相關的挑戰同樣在減少。調查顯示，回答有資料問題和隱私疑慮的公司數量減少，對於模型訓練資料不足的疑慮也降低。這些改善反映出業界的專業知識與資料管理實務正在不斷增加。
+隨著金融服務公司分配預算並更加擅長管理資料，他們可以更好地利用 AI 來提高跨業務單位的營運效率、安全性和進行創新。",AI 帶來亮眼報酬：調查結果揭示金融業最新技術趨勢,https://blogs.nvidia.com.tw/blog/ai-in-financial-services-survey-2025/,"The Financial Services industry is marking an important milestone in its adoption of Artificial Intelligence (AI), with major organizations taking their first steps in testing and experimenting with AI-driven business results.
+
+NVIDIA's fifth annual survey of the State of AI in Financial Services found that financial institutions have integrated AI across all functions, focusing on core applications, a sign that AI capabilities and expertise have increased dramatically.
+
+AI boosts revenue and saves costs
+
+Companies investing in AI are seeing tangible benefits, including increased revenue and cost savings.
+
+About 70% of respondents say AI has driven 5% or more revenue growth, while those seeing 10% to 20% revenue growth make up a significantly larger proportion. Similarly, more than 60% of respondents say AI has helped reduce annual costs by 5% or more.
+
+A quarter of respondents are planning to use AI to create new business opportunities and revenue streams.
+
+Optimizing trading and investment portfolios delivers the highest ROI
+
+24% of respondents cite generative AI use cases as the most prevalent, followed by customer experience and engagement at 21%. These numbers highlight the measurable, real-world benefits AI is bringing to the industry as it transforms critical business areas and drives financial returns.
+
+Overcoming the hurdles to AI success
+
+About half of management respondents say they have deployed their first generative AI service or application, while 28% plan to deploy within the next six months. The number of respondents answering that they lack an AI budget has decreased by 50%, a sign that investment in AI development and resource allocation is increasing.
+
+The challenges associated with early exploration of AI are decreasing as well. The survey shows a decrease in the number of respondents citing data issues and privacy concerns, as well as concerns about insufficient training data. These improvements reflect the growing expertise and data management practices in the industry.
+
+As financial services companies allocate budgets and become more adept at managing data, they will be better equipped to leverage AI to improve operational efficiency, security, and innovation across business units.","The financial services industry is reaching a significant milestone in the use of artificial intelligence (AI), as major organizations move beyond testing and experimentation to successfully use AI to drive business outcomes. 
+
+NVIDIA's fifth annual report on the ""State of AI in Financial Services"" shows that financial institutions have integrated various AI initiatives to focus on core applications, marking a substantial improvement in AI capabilities and proficiency. 
+
+AI helps increase revenue and reduce costs. Companies investing in AI are seeing tangible benefits, including increased revenue and cost savings. Nearly 70% of respondents reported that AI has led to revenue growth of 5% or more, with a significantly higher percentage of respondents seeing revenue growth of 10% to 20%. Additionally, over 60% of respondents indicated that AI has helped reduce annual costs by 5% or more. Nearly a quarter of respondents are planning to use AI to create new business opportunities and revenue streams. 
+
+Trading and portfolio optimization is the highest ROI-generating AI use case, accounting for 25% of responses, followed by customer experience and engagement at 21%. These figures highlight AI's measurable real-world benefits in transforming key business areas and driving financial gains.
+
+Overcoming barriers to AI success. Half of the management-level respondents stated that they have already deployed at least one generative AI service or application, with another 28% planning to deploy in the next six months. The number of respondents citing lack of AI budget has halved, indicating increased investment in AI development and resource allocation.
+
+Challenges associated with early AI exploration are similarly decreasing. The survey shows a reduction in the number of companies reporting data issues and privacy concerns, and worries about insufficient training data for models have also diminished. These improvements reflect the growing expertise and data management practices in the industry.
+
+As financial services companies allocate budgets and become more adept at managing data, they can better leverage AI to enhance operational efficiency, security, and innovation across business units."
+1,1,"Generative AI Powers More Use Cases
+After data analytics, generative AI has emerged as the second-most-used AI workload in the financial services industry. The applications of the technology have expanded significantly, from enhancing customer experience to optimizing trading and portfolio management.
+Notably, the use of generative AI for customer experience, particularly via chatbots and virtual assistants, has more than doubled, rising from 25% to 60%. This surge is driven by the increasing availability, cost efficiency and scalability of generative AI technologies for powering more sophisticated and accurate digital assistants that can enhance customer interactions.
+More than half of the financial professionals surveyed are now using generative AI to enhance the speed and accuracy of critical tasks like document processing and report generation.
+Financial institutions are also poised to benefit from agentic AI systems that harness vast amounts of data from various sources and use sophisticated reasoning to autonomously solve complex, multistep problems. Banks and asset managers can use agentic AI systems to enhance risk management, automate compliance processes, optimize investment strategies and personalize customer services.
+Advanced AI Drives Innovation
+Recognizing the transformative potential of AI, companies are taking proactive steps to build AI factories — specially built accelerated computing platforms equipped with full-stack AI software — through cloud providers or on premises. This strategic focus on implementing high-value AI use cases is crucial to enhancing customer service, boosting revenue and reducing costs.
+By tapping into advanced infrastructure and software, companies can streamline the development and deployment of AI models and position themselves to harness the power of agentic AI.
+With industry leaders predicting at least 2x ROI on AI investments, financial institutions remain highly motivated to implement their highest-value AI use cases to drive efficiency and innovation.
+Download the full report to learn more about how financial services companies are using accelerated computing and AI to transform services and business operations.","生成式 AI 驅動更多使用案例
+繼資料分析之後，生成式 AI 已經成為金融服務業裡第二大宗的 AI 工作負載。這項技術的應用範圍已大幅擴展，從提升客戶體驗到最佳化交易和投資組合管理。
+值得注意的是，生成式 AI 在客戶體驗方面的應用，特別是透過聊天機器人和虛擬助理，數量增加了一倍以上，從 25% 上升到 60%。這樣大幅成長的趨勢是基於生成式 AI 技術的可用性、成本效率和可擴展性不斷提高，能夠驅動更複雜、更精準的數位助理，從而提升客戶互動情況。
+半數以上受訪的金融專業人員現正使用生成式 AI 技術，以提高處理文件和產生報告等重要工作的速度和準確性。
+金融機構也準備好從代理型 AI 中受惠，代理型 AI 系統是指利用各種來源的大量資料，並使用複雜的推理流程自主解決複雜的多步驟問題。銀行和資產管理公司可以使用代理型 AI 系統來加強管理風險、自動化合規流程、最佳化投資策略，還有提供個人化的客戶服務。
+先進的 AI 推動創新
+在意識到 AI 的轉型潛力後，企業正積極採取措施，透過與雲端服務供應商合作或是在地端建立 AI 工廠，這些 AI 工廠是專門打造的加速運算平台，配備全端的 AI 軟體。企業在策略上特別鎖定實施高價值的 AI 使用案例，這對於提升客戶服務、增加收入與降低成本來說至關重要。
+企業利用先進的基礎設施和軟體，可以簡化 AI 模型的開發和部署，並在善加發揮代理型 AI 力量方面站穩腳步。
+由於業界領導業者預測 AI 投資的投資報酬率至少為兩倍，因此金融機構仍有很大動力去實現其最高價值的 AI 使用案例，以推動效率和創新。
+下載完整報告，進一步瞭解金融服務公司如何利用加速運算和 AI 來改變服務和業務運作。",AI 帶來亮眼報酬：調查結果揭示金融業最新技術趨勢,https://blogs.nvidia.com.tw/blog/ai-in-financial-services-survey-2025/,"Generative AI Drives More Use Cases
+Following data analytics, generative AI has become the second largest AI workload in the financial services industry. The scope of this technology's application has expanded significantly, from enhancing customer experience to optimizing transactions and investment portfolio management.
+
+What's worth noting is that generative AI's application in customer experience, particularly through chatbots and virtual assistants, has increased by more than double, from 25% to 60%. This significant growth trend is based on the increasing availability, cost-effectiveness, and scalability of generative AI technology, which can drive more complex and precise digital assistants, thereby enhancing customer interactions.
+
+More than half of the surveyed financial professionals are currently using generative AI technology to improve the speed and accuracy of important work such as processing documents and generating reports.
+
+Financial institutions are also ready to benefit from agent-based AI, which refers to the use of large amounts of data from various sources and complex reasoning processes to autonomously solve complex multi-step problems. Banks and asset management companies can use agent-based AI systems to strengthen risk management, automate compliance processes, optimize investment strategies, and provide personalized customer services.
+
+Advanced AI drives innovation
+Following the realization of AI's transformative potential, companies are taking proactive measures by partnering with cloud service providers or establishing AI factories on-site. These AI factories are specialized platforms for accelerated computing, equipped with end-to-end AI software. Companies strategically focus on implementing high-value AI use cases, which are critical for enhancing customer service, increasing revenue, and reducing costs.
+
+With advanced infrastructure and software, companies can simplify the development and deployment of AI models and establish a strong footing in leveraging the power of agent-based AI.
+
+Due to industry leaders predicting that AI investments will yield at least a 2x return on investment, financial institutions still have significant motivation to achieve their highest-value AI use cases to drive efficiency and innovation.
+
+Download the full report to learn more about how financial services companies use accelerated computing and AI to change their services and business operations.","Generative AI Drives More Use Cases
+
+Following data analytics, generative AI has become the second largest AI workload in the financial services industry. The scope of this technology has significantly expanded, from enhancing customer experience to optimizing trading and portfolio management.
+
+Notably, the application of generative AI in customer experience, particularly through chatbots and virtual assistants, has more than doubled, rising from 25% to 60%. This significant growth trend is based on the increasing availability, cost efficiency, and scalability of generative AI technology, enabling the development of more complex and accurate digital assistants, thereby enhancing customer interactions.
+
+More than half of the financial professionals surveyed are currently using generative AI technology to improve the speed and accuracy of critical tasks such as document processing and report generation.
+
+Financial institutions are also poised to benefit from agent-based AI. These systems utilize vast amounts of data from various sources and employ complex reasoning processes to autonomously solve multi-step complex problems. Banks and asset management companies can use agent-based AI systems to strengthen risk management, automate compliance processes, optimize investment strategies, and provide personalized customer services.
+
+Advanced AI Drives Innovation
+
+Recognizing the transformative potential of AI, businesses are actively taking measures by partnering with cloud service providers or establishing local AI factories, which are specialized accelerated computing platforms equipped with end-to-end AI software. Strategically, companies focus particularly on implementing high-value AI use cases that are crucial for improving customer service, increasing revenue, and reducing costs.
+
+By leveraging advanced infrastructure and software, businesses can streamline the development and deployment of AI models and firmly establish themselves in utilizing the power of agent-based AI.
+
+As industry leaders anticipate a return on investment in AI of at least two times, financial institutions have strong motivation to realize their highest-value AI use cases to drive efficiency and innovation.
+
+Download the full report to learn more about how financial services companies are using accelerated computing and AI to transform services and business operations."
+4,0,"Generative AI and foundation models let autonomous machines generalize beyond the operational design domains on which they’ve been trained. Using new AI techniques such as tokenization and large language and diffusion models, developers and researchers can now address longstanding hurdles to autonomy.
+These larger models require massive amounts of diverse data for training, fine-tuning and validation. But collecting such data — including from rare edge cases and potentially hazardous scenarios, like a pedestrian crossing in front of an autonomous vehicle (AV) at night or a human entering a welding robot work cell — can be incredibly difficult and resource-intensive.
+To help developers fill this gap, NVIDIA Omniverse Cloud Sensor RTX APIs enable physically accurate sensor simulation for generating datasets at scale. The application programming interfaces (APIs) are designed to support sensors commonly used for autonomy — including cameras, radar and lidar — and can integrate seamlessly into existing workflows to accelerate the development of autonomous vehicles and robots of every kind.
+Omniverse Sensor RTX APIs are now available to select developers in early access. Organizations such as Accenture, Foretellix, MITRE and Mcity are integrating these APIs via domain-specific blueprints to provide end customers with the tools they need to deploy the next generation of industrial manufacturing robots and self-driving cars.
+Powering Industrial AI With Omniverse Blueprints
+In complex environments like factories and warehouses, robots must be orchestrated to safely and efficiently work alongside machinery and human workers. All those moving parts present a massive challenge when designing, testing or validating operations while avoiding disruptions.
+Mega is an Omniverse Blueprint that offers enterprises a reference architecture of NVIDIA accelerated computing, AI, NVIDIA Isaac and NVIDIA Omniverse technologies. 
+Enterprises can use it to develop digital twins and test AI-powered robot brains that drive robots, cameras, equipment and more to handle enormous complexity and scale.
+Integrating Omniverse Sensor RTX, the blueprint lets robotics developers simultaneously render sensor data from any type of intelligent machine in a factory for high-fidelity, large-scale sensor simulation.
+With the ability to test operations and workflows in simulation, manufacturers can save considerable time and investment, and improve efficiency in entirely new ways.","生成式人工智慧（AI）和基礎模型讓自主機器能夠超越它們所接受訓練的操作設計領域。開發人員和研究人員使用標記化（tokenization）及大型語言和擴散模型等嶄新 AI 技術，現在可以解決一直以來在自主領域方面的各項障礙。
+需要使用大量相異的資料來訓練、微調與驗證這些大型模型。不過收集這些資料（包括從罕見的邊緣情況和潛在危險情境中收集資料，例如行人在夜間橫越自動駕駛車前方，或是人類進入焊接機器人工作單元）可能非常困難，又得耗費不少資源。
+為了協助開發人員填補這個缺口，NVIDIA Omniverse Cloud Sensor RTX API 提供了物理精確的感測器模擬，用於大規模生成資料集。這些應用程式介面（API）用於支援常用於自主機器上的感測器，包括攝影機、雷達與光達，且能完美與現有的工作流程進行整合，以加快開發各種自動駕駛車輛與機器人。
+現已開放部分開發人員搶先體驗
+Omniverse Sensor RTX API。埃森哲（Accenture）、Foretellix、MITRE 和 Mcity等企業正透過特定領域藍圖整合這些 API，為終端客戶提供部署下一代工業製造機器人和自動駕駛車所需的工具。
+使用 Omniverse Blueprints 為工業 AI 提供動力
+在工廠和倉庫等複雜環境中，機器人必須被精心協調，才能安全高效率地與機器和人類工作者並肩作業。在設計、測試或驗證操作，又要避免中斷作業時，所有這些移動部件都會帶來巨大的挑戰。
+Mega 是一個 Omniverse Blueprint ，可為企業提供 NVIDIA 加速運算、AI、NVIDIA Isaac 及 NVIDIA Omniverse 技術的參考架構。企業可以用它開發數位孿生模型，測試由 AI 驅動的機器人大腦，而這些大腦驅動著機器人、攝影機、設備等項目，以處理極為複雜又大量的作業。
+這個整合了 Omniverse Sensor RTX 的藍圖可以讓機器人開發人員同時渲染工廠內任何類型智慧機器的感測器資料，實現高保真、大規模的感測器模擬。
+隨著能夠在模擬環境裡測試操作和工作流程，製造商可以省下大量時間和投資，以全新方式提高作業效率。",建造更聰明的自主機器：NVIDIA 宣布 Omniverse Sensor RTX 推出搶先體驗活動,https://blogs.nvidia.com.tw/blog/omniverse-sensor-rtx-autonomous-machines/,"Generative artificial intelligence (AI) and foundation models enable autonomous machines to surpass the operational design domains in which they were trained. Developers and researchers are using tokenization and large language and diffusion models and other AI techniques to overcome long-standing challenges in the autonomous domain.
+
+Training, fine-tuning, and validating these large models requires an abundance of diverse data. However, collecting this data, including from rare edge cases and potentially hazardous scenarios, such as pedestrians crossing in front of autonomous vehicles at night or humans entering a welding robot workcell, can be extremely difficult and costly.
+
+To help fill this gap, NVIDIA Omniverse Cloud Sensor RTX API offers physically accurate sensor simulation for large-scale data generation. These APIs are designed to support sensors commonly found on autonomous machines, including cameras, radar, and lidar, and can be seamlessly integrated into existing workflows to accelerate the development of various autonomous vehicles and robots.
+
+Omniverse Sensor RTX API is now available for early adopters. Companies like Accenture, Foretellix, MITRE, and Mcity are integrating these APIs into specific domain blueprints to provide end customers with the tools needed to deploy next-generation industrial manufacturing robots and autonomous vehicles.
+
+Harnessing Omniverse Blueprints for Industrial AI
+
+In complex environments such as factories and warehouses, robots must be carefully coordinated to operate safely and efficiently alongside machines and human workers. Designing, testing, or verifying operations while avoiding downtime brings a multitude of challenges to these moving parts.
+
+Mega is an Omniverse Blueprint that provides a reference architecture with NVIDIA acceleration, AI, NVIDIA Isaac, and NVIDIA Omniverse technologies for enterprises. With it, companies can develop digital twins, test AI-driven robot brains, and deploy these brains to robots, cameras, equipment, and other assets to handle complex and massive workloads.
+
+This integrated Blueprint with Omniverse Sensor RTX enables robotics developers to simultaneously render sensor data for any type of intelligent machine within a factory, achieving high-fidelity, large-scale sensor simulation.
+
+By testing operations and workflows in a simulated environment, manufacturers can save significant time and investment and improve operational efficiency in new ways.","Generative Artificial Intelligence (AI) and foundational models enable autonomous machines to operate beyond their training design domains. Developers and researchers use innovative AI technologies like tokenization, large language, and diffusion models to overcome long-standing barriers in the field of autonomy. 
+
+Training, fine-tuning, and validating these large models require a vast amount of diverse data. However, collecting such data, including from rare edge cases and potentially dangerous situations—like pedestrians crossing in front of autonomous vehicles at night or humans entering a welding robot's workspace—can be challenging and resource-intensive.
+
+To help developers bridge this gap, the NVIDIA Omniverse Cloud Sensor RTX API offers physically accurate sensor simulation for large-scale dataset generation. These APIs are designed to support sensors commonly used on autonomous machines, including cameras, radars, and LiDARs, and seamlessly integrate with existing workflows to accelerate the development of various autonomous vehicles and robots.
+
+The Omniverse Sensor RTX API is now available for early access by selected developers. Companies like Accenture, Foretellix, MITRE, and Mcity are integrating these APIs through domain-specific blueprints to equip end customers with the tools needed to deploy next-generation industrial manufacturing robots and autonomous vehicles.
+
+Powering industrial AI with Omniverse Blueprints
+In complex environments like factories and warehouses, robots must be meticulously coordinated to work safely and efficiently alongside machines and human workers. Designing, testing, or validating operations without interrupting ongoing processes presents substantial challenges with these moving parts.
+
+Mega is an Omniverse Blueprint that provides enterprises with a reference architecture for NVIDIA accelerated computing, AI, NVIDIA Isaac, and NVIDIA Omniverse technologies. Enterprises can use it to develop digital twin models and test AI-driven robotic brains, which power robots, cameras, devices, and more to handle highly complex and large-scale operations.
+
+This blueprint, integrated with Omniverse Sensor RTX, allows robot developers to simultaneously render sensor data from any type of intelligent machine within a factory, achieving high-fidelity, large-scale sensor simulation.
+
+By testing operations and workflows in simulated environments, manufacturers can save significant time and investment, enhancing operational efficiency in entirely new ways."
+4,1,"International supply chain solutions company KION Group and Accenture are using the Mega blueprint to build Omniverse digital twins that serve as virtual training and testing environments for industrial AI’s robot brains, tapping into data from smart cameras, forklifts, robotic equipment and digital humans.
+The robot brains perceive the simulated environment with physically accurate sensor data rendered by the Omniverse Sensor RTX APIs. They use this data to plan and act, with each action precisely tracked with Mega, alongside the state and position of all the assets in the digital twin. 
+With these capabilities, developers can continuously build and test new layouts before they’re implemented in the physical world.
+Driving AV Development and Validation
+Autonomous vehicles have been under development for over a decade, but barriers in acquiring the right training and validation data and slow iteration cycles have hindered large-scale deployment.
+To address this need for sensor data, companies are harnessing the NVIDIA Omniverse Blueprint for AV simulation, a reference workflow that enables physically accurate sensor simulation. The workflow uses Omniverse Sensor RTX APIs to render the camera, radar and lidar data necessary for AV development and validation.
+AV toolchain provider Foretellix has integrated the blueprint into its
+Foretify AV development toolchain
+to transform object-level simulation into physically accurate sensor simulation.
+The Foretify toolchain can generate any number of testing scenarios simultaneously. By adding sensor simulation capabilities to these scenarios, Foretify can now enable  developers to evaluate the completeness of their AV development, as well as train and test at the levels of fidelity and scale needed to achieve large-scale and safe deployment. In addition, Foretellix will use the newly announced
+NVIDIA Cosmos platform to generate an even greater diversity of scenarios for verification and validation.
+Nuro, an autonomous driving technology provider with one of the largest level 4 deployments in the U.S., is using the Foretify toolchain to train, test and validate its self-driving vehicles before deployment.
+In addition, research organization MITRE is collaborating with the University of Michigan’s Mcity testing facility to build a digital AV validation framework for regulatory use, including a digital twin of Mcity’s 32-acre proving ground for autonomous vehicles. The project uses the AV simulation blueprint to render physically accurate sensor data at scale in the virtual environment, boosting training effectiveness.
+The future of robotics and autonomy is coming into sharp focus, thanks to the power of high-fidelity sensor simulation. Learn more about these solutions at CES by visiting Accenture at Ballroom F at the Venetian and Foretellix booth 4016 in the West Hall of Las Vegas Convention Center.
+Learn more about the latest in automotive and generative AI technologies by joining
+NVIDIA at CES.","國際供應鏈解決方案公司凱傲集團（KION Group）與埃森哲利用來自智慧攝影機、堆高機、機器人設備和數位人類的資料，使用 Mega 藍圖建立 Omniverse 數位孿生，作為工業AI機器人大腦的虛擬訓練和測試環境。
+機器人大腦透過 Omniverse Sensor RTX API 渲染的物理精確感測器資料來感知模擬環境。機器人使用這些資料來計劃和採取行動，並透過 Mega 精準追蹤每一個動作，以及數位孿生中所有資產的狀態和位置。借助這些功能，開發人員可以在真正部署至實體環境裡之前，不斷建立和測試新配置。
+推動開發與驗證自動駕駛車
+自動駕駛車輛已開發超過十多年，但在取得正確的訓練與驗證資料方面所遇到的阻礙，還有緩慢的迭代週期，都阻礙了大規模部署。
+為了滿足對感測器資料的這種需求，各家公司利用 NVIDIA Omniverse Blueprint for AV simulation，這是一個實現物理精確感測器模擬的參考工作流程。這個工作流程使用 Omniverse Sensor RTX API 來渲染出開發與驗證自動駕駛汽車所需的攝影機、雷達與光達資料。
+自動駕駛汽車工具鏈供應商 Foretellix 已經把這個藍圖納入該公司的 Foretify 自動駕駛車開發工作鏈，將物件級模擬轉換為物理精準感測器模擬。
+Foretify 工具鏈可以同時產生任意數量的測試情境。Foretify 在這些情境中加入感測器模擬功能，開發人員便能評估自己在開發自動駕駛車方面的完整性，並以實現大規模安全部署所需的保真度和規模水平進行訓練和測試。。Foretellix 還將使用最新發表的
+NVIDIA Cosmos 平台，產生更多樣化的情境進行確認與驗證。
+自動駕駛技術提供商 Nuro 是美國規模最大的 level 4 部署業者之一，使用 Foretify 工具鏈在部署前對其自動駕駛車輛進行訓練、測試和驗證。
+再者，研究機構 MITRE 與密西根大學的 Mcity 測試設施合作，建立供主管機關使用的數位自動駕駛車驗證框架，包括 Mcity 32 英畝自動駕駛車試驗場的數位孿生模型。這項合作案使用 自動駕駛車 模擬藍圖，在虛擬環境中大規模渲染出物理精確的感測器資料，以提升訓練成效。
+得益於高保真感測器模擬技術，機器人與自動化的未來正逐漸成為人們關注的焦點。如需更深入瞭解 CES 大會上這些解決方案的資訊，請造訪埃森哲位於拉斯維加斯威尼斯人F展廳的攤位，以及 Foretellix 位於拉斯維加斯展覽中心西館 4016 號的展位。
+欲了解最新的汽車與生成式 AI 技術，參加 NVIDIA 在 CES 大會的各項活動。",建造更聰明的自主機器：NVIDIA 宣布 Omniverse Sensor RTX 推出搶先體驗活動,https://blogs.nvidia.com.tw/blog/omniverse-sensor-rtx-autonomous-machines/,"KION Group, a leading global intralogistics company, and Accenture have leveraged data from smart cameras, forklifts, robotics equipment, and digital humans to create an Omniverse digital twin, powered by Mega Graph, as a virtual training and testing environment for industrial AI robots. The robot brain perceives the simulated environment through physically accurate sensor data rendered by the Omniverse Sensor RTX API. The robot uses this data to plan and act, and through Mega, it precisely tracks every action and the state and location of all assets within the digital twin. By leveraging these capabilities, developers can continuously build and test new configurations before deploying them to physical environments.
+
+Driving Autonomous Vehicle Development
+Autonomous vehicles have been in development for over a decade, but have been hindered by the difficulty in obtaining accurate training and validation data, as well as the slow iteration cycle. To meet this demand for sensor data, companies have turned to NVIDIA Omniverse Blueprint for AV simulation, a reference workflow that realizes physically accurate sensor simulation. This workflow uses the Omniverse Sensor RTX API to render camera, radar, and lidar data needed for developing and validating autonomous vehicles.
+
+Autonomous Vehicle Tools and Services Provider Foretellix has integrated this blueprint into its Foretify autonomous vehicle development workflow, converting object-level simulation to physically accurate sensor simulation. The Foretify toolchain can generate an arbitrary number of test scenarios, with sensor simulation added in these scenarios, allowing developers to evaluate their completeness in developing autonomous vehicles, and train and test at the fidelity and scale required for large-scale deployments. Foretellix is also using the newly released NVIDIA Cosmos platform to generate more diverse scenarios for validation and verification.
+
+Nuro, a leading provider of autonomous technology and one of the largest Level 4 deployment operators in the US, is using the Foretify toolchain to train, test, and validate its autonomous vehicles before deployment.
+
+In addition, research institutions MITRE and the University of Michigan's Mcity test facility are collaborating to establish a digital autonomous vehicle validation framework for government agencies, including a digital twin model of the Mcity 32-acre autonomous vehicle test field. This collaboration is using the autonomous vehicle simulation blueprint to render physically accurate sensor data at scale in a virtual environment, improving training effectiveness.
+
+Thanks to high-fidelity sensor simulation technology, the future of robots and automation is becoming a growing focus area. For more information on these solutions showcased at CES, visit Accenture's booth at the Venetian in Las Vegas, as well as Foretellix's booth #4016 at the West Hall of the Las Vegas Convention Center. For the latest on automotive and generative AI technology, attend NVIDIA's various activities at CES.","The international supply chain solutions company KION Group, in collaboration with Accenture, is using data from smart cameras, forklifts, robotics equipment, and digital humans to build an Omniverse digital twin with Mega blueprints. This serves as a virtual training and testing environment for the industrial AI robot brain. The robot brain perceives the simulated environment through physically accurate sensor data rendered by the Omniverse Sensor RTX API. Robots use this data to plan and take actions, with Mega accurately tracking every movement and the status and location of all assets in the digital twin. These features allow developers to continuously build and test new configurations before deploying them in a physical environment. 
+
+Advancing the Development and Validation of Autonomous Vehicles
+Autonomous vehicles have been in development for over a decade, but obstacles in obtaining the correct training and validation data, along with slow iteration cycles, have hindered large-scale deployment. To meet the demand for sensor data, companies are utilizing the NVIDIA Omniverse Blueprint for AV simulation, a reference workflow that realizes physically accurate sensor simulations. This workflow uses the Omniverse Sensor RTX API to render the camera, radar, and lidar data needed for developing and validating autonomous vehicles.
+
+Foretellix, an autonomous vehicle toolchain supplier, has incorporated this blueprint into the company’s Foretify autonomous vehicle development toolchain, transforming object-level simulation into physically precise sensor simulation. The Foretify toolchain can generate any number of test scenarios simultaneously. By incorporating sensor simulation into these scenarios, developers can assess the integrity of their autonomous vehicle development with the fidelity and scale required for large-scale safe deployment. Foretellix will also use the newly released NVIDIA Cosmos platform to generate more diverse scenarios for validation and verification. 
+
+Nuro, one of the largest level 4 deployment operators in the U.S., uses the Foretify toolchain to train, test, and validate their autonomous vehicles before deployment.
+
+Additionally, the research institution MITRE is collaborating with the University of Michigan's Mcity testing facility to create a digital verification framework for autonomous vehicles for regulatory authorities, including a digital twin model of Mcity's 32-acre autonomous vehicle testing grounds. This collaboration uses the autonomous vehicle simulation blueprint to render physically accurate sensor data on a large scale in virtual environments to enhance training effectiveness.
+
+Thanks to high-fidelity sensor simulation technology, the future of robotics and automation is becoming a focal point of interest. For more information about these solutions presented at the CES conference, visit Accenture's booth at the Venetian F Hall in Las Vegas and Foretellix's booth at the Las Vegas Convention Center West Hall, booth 4016. To stay updated on the latest automotive and generative AI technologies, participate in NVIDIA's activities at the CES conference."
+6,0,"Greetings, Traveler. Prepare for adventure.
+Genshin Impact, the popular open-world action role-playing game, is leaving limited beta and launching for all GeForce NOW members next week.
+Gamers can get their game on today with the six total games joining the GeForce NOW library.
+As announced last week, Warhammer 40,000: Darktide is coming to the cloud at launch — with GeForce technology. This September, members will be able to leap thousands of years into the future to the time of the Space Marines, streaming on GeForce NOW with NVIDIA DLSS and more.
+Plus, the 2.0.41 GeForce NOW app update brings a highly requested feature: in-stream copy-and-paste support from the clipboard while streaming from the PC and Mac apps — so there’s no need to enter a long, complex password for the digital store. Get to your games even faster with this new capability.
+GeForce NOW is also giving mobile gamers more options by bringing the perks of RTX 3080 memberships and PC gaming at 120 frames per second to all devices with support for 120Hz phones. The capability is rolling out in the coming weeks.
+Take a Trip to Teyvat
+After the success of a limited beta and receiving great feedback from members, Genshin Impact is coming next week to everyone streaming on GeForce NOW.
+Embark on a journey as a traveler from another world, stranded in the fantastic land of Teyvat. Search for your missing sibling in a vast continent made up of seven nations. Master the art of elemental combat and build a dream team of over 40 uniquely skilled playable characters – like the newest additions of Yelan and Kuki Shinobu – each with their own rich stories, personalities and combat styles.
+Experience the immersive campaign, dive deep into rich quests alongside iconic characters and complete daily challenges. Charge head-on into battles solo or invite friends to join the adventures. The world is constantly expanding, so bring it wherever you go across devices, streaming soon to underpowered PCs, Macs and Chromebooks on GeForce NOW.
+RTX 3080 members can level up their gaming for the best experience by streaming in 4K resolution and 60 frames per second on the PC and Mac apps.
+Let the Gaming Commence
+All of the action this GFN Thursday kicks off with six new games arriving on the cloud. Members can also gear up for Rainbow Six Siege Year 7 Season 2.
+Get ready for a new Operator, Team Deathmatch map and more in “Rainbow Six Siege” Year 7 Season 2.
+Members can look for the following streaming this week:Chivalry 2(New release on Steam) Starship Troopers – Terran Command(New release on Steam and Epic Games Store) Builder Simulator(Steam) Supraland(Free on Epic Games Store) The Legend of Heroes: Trails of Cold Steel II(Steam) POSTAL: Brain Damaged(Steam)
+Finally, members still have a chance to stream the PC Building Simulator 2 open beta before it ends on Monday, June 20. Experience deeper simulation, an upgraded career mode and powerful new customization features to bring your ultimate PC to life.","旅人你好，準備踏上冒險之旅吧。熱門開放世界動作角色扮演遊戲《原神》即將結束限量公測版，並將於下週推出，供所有 GeForce NOW 會員遊玩。
+還有六款遊戲現已加入 GeForce NOW 遊戲庫，供玩家即刻暢玩。
+正如上週公告，《戰鎚 40K：黑潮 (Warhammer 40,000:  Darktide)》即將於雲端推出，由 GeForce 技術支援。今年九月，會員將能橫跨數千年後的未來，進入太空海軍陸戰隊時代，遊戲將可於 GeForce NOW 上串流。
+前往提瓦特《原神》限時公測版大獲成功，得到會員的極佳回饋，並將於下週開始在 GeForce NOW 上開放串流，供所有玩家遊玩。
+化身來自另一世界的旅人踏上冒險之途，流連於提瓦特的奇幻土地。在由七個國家組成的寬廣大陸尋找失蹤手足。掌握元素戰鬥的藝術，打造一支夢幻團隊，40 多位角色均具備獨一無二的技能，例如最新加入的夜蘭 (Yelan) 和久岐忍 (Kuki Shinobu)，他們各自都有豐富的故事、個性和戰鬥風格。
+在《Chasm》的2.7版「荒夢藏虞淵(Hidden Dreams in the Depths)」更新中，探索故事深處的奧秘。
+體驗身歷其境的戰役、與經典角色一同深入探索豐富任務並完成每日挑戰。衝鋒陷陣單打獨鬥，或邀請好友加入冒險。世界正在持續擴張，所以無論身處何處都能跨裝置使用，快速在低效能的 PC、Mac和 Chromebook 上透過 GeForce NOW串流遊玩。
+遊戲開始
+本週 GFN 以六款於雲端推出的新遊戲揭開序幕。會員也可以準備迎接《虹彩六號：圍攻行動(Rainbow Six Siege)》第 7 年第 2 季。
+準備好迎接《虹彩六號：圍攻行動 （Rainbow Six Siege） 》第 7 年第 2 季新加入的戰鬥員、團隊殊死戰 (Team Deathmatch) 地圖等更多內容。
+會員可於本週稍後期待以下遊戲開放串流：《騎士精神 2 (Chivalry 2)》(於 Steam 全新發佈)《星艦戰將：人類總動員(Starship Troopers – Terran Command)》(於 Steam 與 Epic Games Store 全新發佈)《Builder Simulator》(Steam)《Supraland》(Epic Games Store 開放免費遊玩)《英雄傳說閃之軌跡 II (The Legend of Heroes: Trails of Cold Steel II)》(Steam)《喋血街頭：腦損(POSTAL: Brain Damaged) 》(Steam)
+最後，會員仍有機會在 6 月 20 日星期一結束前，串流遊玩《PC Builder Simulator 2》公測版。體驗更深入的模擬效果、經過升級的生涯模式和強大的全新自訂功能，讓你的終極 PC 栩栩如生。",願望成真：《原神 (Genshin Impact) 》即將於 GeForce NOW 聯盟 Taiwan Mobile 雲端遊戲服務推出,https://blogs.nvidia.com.tw/blog/geforce-now-thursday-june-16/,"A traveler, are you ready to embark on an adventure? The highly-anticipated open-world action role-playing game Genshin Impact is nearing the end of its limited beta test and will be launched next week for all GeForce NOW members to play.
+
+Six new games have been added to the GeForce NOW game library, available to play now.
+
+As previously announced, Warhammer 40,000: Darktide will be launched on the cloud with GeForce technology support. In September, members will be able to travel to a future where space marines rule, with the game streaming on GeForce NOW.
+
+Genshin Impact's limited beta test was a huge success, with excellent feedback from members, and will begin streaming on GeForce NOW next week for all players to play.
+
+As a traveler from another world, embark on an adventure to explore the fantastical lands of Teyvat, searching for a missing companion. Master the art of elemental combat, build a dream team, and choose from over 40 characters, each with unique skills, such as the new Yelan and Kuki Shinobu, with their own stories, personalities, and combat styles.
+
+In Chasm's 2.7 update ""Hidden Dreams in the Depths,"" explore the secrets of the story.
+
+Experience immersive campaigns, explore rich missions with classic characters, and complete daily challenges. Engage in single-player or invite friends to join the adventure. The world is constantly expanding, so you can use it on any device, quickly streaming on low-powered PCs, Macs, and Chromebooks through GeForce NOW.
+
+New Games
+
+This week, GFN kicks off with six new games being launched on the cloud. Members can also prepare for Rainbow Six Siege's Season 7, Year 2.
+
+Get ready for Rainbow Six Siege's Season 7, Year 2, with new operators, Team Deathmatch maps, and more.
+
+Members can expect the following games to be streaming on GeForce NOW this week:
+
+Chivalry 2 (newly released on Steam)
+Starship Troopers – Terran Command (newly released on Steam and Epic Games Store)
+Builder Simulator (Steam)
+Supraland (Epic Games Store free to play)
+The Legend of Heroes: Trails of Cold Steel II (Steam)
+POSTAL: Brain Damaged (Steam)
+
+Lastly, members still have a chance to stream PC Builder Simulator 2's public test before it ends on June 20th. Experience more immersive simulation, upgraded career mode, and powerful new customization features, bringing your ultimate PC to life.","Hello, traveler, get ready to embark on an adventure. The popular open-world action RPG ""Genshin Impact"" is about to conclude its limited beta version and will be launched next week for all GeForce NOW members to play.
+
+Additionally, six new games have now joined the GeForce NOW library for players to enjoy immediately. As announced last week, ""Warhammer 40,000: Darktide"" will soon be launched on the cloud, supported by GeForce technology. In September this year, members will be able to enter the era of Space Marines in the far future, with the game available for streaming on GeForce NOW.
+
+The limited-time beta test for ""Genshin Impact"" in Teyvat was a great success, receiving excellent feedback from members. Starting next week, the game will be available for streaming on GeForce NOW for all players.
+
+Transform into a traveler from another world and embark on an adventure in the fantasy land of Teyvat. Explore the vast continent made up of seven nations in search of your missing sibling. Master the art of elemental combat and build a dream team with over 40 characters, each with unique skills, such as the newly added Yelan and Kuki Shinobu, who each have rich stories, personalities, and combat styles.
+
+In the Version 2.7 update ""Hidden Dreams in the Depths"" of ""The Chasm,"" uncover the mysteries deep within the story. Experience immersive campaigns, explore richly immersive quests with classic characters, and complete daily challenges. Charge into battle alone or invite friends to join the adventure. The world is continuously expanding, so you can access it across devices, streaming quickly on low-performance PCs, Macs, and Chromebooks via GeForce NOW.
+
+Let the games begin. This week, GFN kicks off with six new cloud-distributed games. Members can also look forward to the 2nd season of the 7th year of ""Rainbow Six Siege.""
+
+Get ready to welcome new operatives in ""Rainbow Six Siege"" Year 7 Season 2, new Team Deathmatch maps, and more content. Members can look forward to streaming the following games later this week: ""Chivalry 2"" (newly released on Steam), ""Starship Troopers – Terran Command"" (newly released on Steam and Epic Games Store), ""Builder Simulator"" (Steam), ""Supraland"" (free to play on Epic Games Store), ""The Legend of Heroes: Trails of Cold Steel II"" (Steam), and ""POSTAL: Brain Damaged"" (Steam).
+
+Lastly, members still have the chance to stream the beta version of ""PC Builder Simulator 2"" before it ends on Monday, June 20. Experience deeper simulation effects, an upgraded career mode, and powerful new customization features that bring your ultimate PC to life."
+8,0,"Autonomous vehicle (AV) development is made possible by three distinct computers:
+NVIDIA DGX systems for training the AI-based stack in the data center, NVIDIA Omniverse running on NVIDIA OVX systems for simulation and synthetic data generation, and the NVIDIA AGX in-vehicle computer to process real-time sensor data for safety.
+Together, these purpose-built, full-stack systems enable continuous development cycles, speeding improvements in performance and safety.
+At the CES trade show, NVIDIA today announced a new part of the equation:
+NVIDIA Cosmos, a platform comprising state-of-the-art generative world foundation models (WFMs), advanced tokenizers, guardrails and an accelerated video processing pipeline built to advance the development of physical AI systems such as AVs and robots.
+With Cosmos added to the three-computer solution, developers gain a data flywheel that can turn thousands of human-driven miles into billions of virtually driven miles — amplifying training data quality.
+“The AV data factory flywheel consists of fleet data collection, accurate 4D reconstruction and AI to generate scenes and traffic variations for training and closed-loop evaluation,” said Sanja Fidler, vice president of AI research at NVIDIA. “Using the NVIDIA Omniverse platform, as well as Cosmos and supporting AI models, developers can generate synthetic driving scenarios to amplify training data by orders of magnitude.”
+“Developing physical AI models has traditionally been resource-intensive and costly for developers, requiring acquisition of real-world datasets and filtering, curating and preparing data for training,” said Norm Marks, vice president of automotive at NVIDIA. “Cosmos accelerates this process with generative AI, enabling smarter, faster and more precise AI model development for autonomous vehicles and robotics.”
+Transportation leaders are using Cosmos to build physical AI for AVs, including:
+Waabi
+, a company pioneering generative AI for the physical world, will use Cosmos for the search and curation of video data for AV software development and simulation.
+Wayve
+, which is developing AI foundation models for autonomous driving, is evaluating Cosmos as a tool to search for edge and corner case driving scenarios used for safety and validation.
+AV toolchain provider Foretellix will use Cosmos, alongside NVIDIA Omniverse Sensor RTX APIs, to evaluate and generate high-fidelity testing scenarios and training data at scale.
+In addition, ridesharing giant Uber is partnering with NVIDIA to accelerate autonomous mobility. Rich driving datasets from Uber, combined with the features of the Cosmos platform and
+NVIDIA DGX Cloud, will help AV partners build stronger AI models even more efficiently.
+Availability
+Cosmos WFMs are now available under an open model license on Hugging Face and the NVIDIA NGC catalog. 
+Cosmos models will soon be available as fully optimized NVIDIA NIM microservices.
+Get started with Cosmos and join NVIDIA at CES.","自動駕駛的發展以三台不同的電腦實現：
+NVIDIA DGX 系統用於在資料中心訓練以人工智慧（AI）為基礎的堆疊，在 NVIDIA OVX 系統上運行的 NVIDIA Omniverse 用於模擬與產生合成資料，而 NVIDIA AGX 車載電腦則用於即時處理感測器產生出的資料以確保安全。
+這些專門建置的全堆疊系統共同推動持續性的開發進程，加快提高效能與安全性。
+NVIDIA 今日在 CES 大會宣布此方程式又加入一個新成員：NVIDIA Cosmos。 這個平台包含最先進的生成世界基礎模型（WFM）、先進的標記器、護欄和加速影片處理管道，專為推動開發自駕車輛與機器人等實體 AI 系統而打造。
+將 Cosmos 加入三台電腦的解決方案，開發人員獲得一個資料飛輪，可以將人類駕駛所累積出的數千哩的里程轉換為數十億哩的虛擬駕駛里程，提高訓練資料的品質。
+NVIDIA AI 研究部門副總裁 Sanja Fidler 表示：「自動駕駛資料工廠的飛輪包括收集車隊資料、精準的 4D 重構與 AI，以產生場景與各種交通路況，用於訓練與閉環評估。開發人員使用 NVIDIA Omniverse 平台以及 Cosmos 和支援的 AI 模型，可以產生合成的行車場景，將訓練資料放大數倍。」
+NVIDIA車用產品副總裁 Norm Marks 表示：「開發人員在開發實體 AI 模型的過程向來是資源密集且成本高昂的工作，需要取得真實世界的資料集，並且篩選、整理和準備訓練資料。Cosmos利用生成式 AI 加快這個過程，更聰明、快速且精確開發用於自動駕駛和機器人的 AI 模型。」
+交通運輸領域領導業者使用 Cosmos 為自動駕駛建立實體 AI，包括：
+Waabi
+為實體世界開創生成式 AI，使用 Cosmos 搜尋和整理影片資料，用於開發和模擬自動駕駛軟體。
+Wayve
+開發適用於自動駕駛的 AI 基礎模型，正在評估 Cosmos，將其作為搜尋用於安全和驗證之邊緣和極端駕駛情況的工具。
+自駕車工具鏈供應商 Foretellix 使用 Cosmos 與 NVIDIA Omniverse Sensor RTX API，以評估和產生大量高擬真度的測試場景及訓練資料。
+此外，乘車服務巨擘 Uber 也將與 NVIDIA 合作，加速推動開發自動���駛移動技術。Uber 提供豐富的駕駛資料集，加上 Cosmos 平台與 NVIDIA DGX Cloud，將協助自駕車合作夥伴更有效率地建立更強大的 AI 模型。
+上市時間
+Cosmos WFM 現已在 Hugging Face 及 NVIDIA NGC 目錄上以開放模型授權的方式提供。Cosmos模型即將以完全最佳化 NVIDIA NIM 微服務的形式提供。
+開始使用 Cosmos、觀看示範，並且參加 NVIDIA 在 CES 大會的活動。",NVIDIA以 Cosmos 世界基礎模型增強適用於自動駕駛的三台電腦解決方案,https://blogs.nvidia.com.tw/blog/three-computer-cosmos-ces/,"The development of autonomous driving technology is achieved through three distinct computers: The NVIDIA DGX system, which is used for training AI-based stacks in the data center and runs NVIDIA Omniverse on the NVIDIA OVX system, which is used for simulating and generating synthetic data. The NVIDIA AGX car computer is used to process sensor data in real-time to ensure safety. These purpose-built end-to-end systems drive continuous development and improvement in performance and safety.
+
+NVIDIA today announced at the CES event that it has added a new member to the equation: NVIDIA Cosmos. This platform includes the most advanced World Baseline Model (WFM) generation, advanced markers, guards, and acceleration of video processing pipelines, which are specifically designed to drive the development of autonomous vehicles and other physical AI systems.
+
+Adding Cosmos to the three computer solution, developers get a data wheel that can turn the miles accumulated by human drivers into tens of billions of virtual driving miles, improving the quality of training data.
+
+NVIDIA AI Research Department Vice President Sanja Fidler said: ""The flywheel of the autonomous driving data factory includes collecting fleet data, accurate 4D reconstruction and AI to generate scenarios and various traffic conditions for training and closed-loop evaluation. Using the NVIDIA Omniverse platform and Cosmos and supported AI models, developers can generate synthetic driving scenarios, expanding training data by orders of magnitude.""
+
+NVIDIA Vice President of Automotive Products Norm Marks said: ""Developers in the development of physical AI models have traditionally been resource-intensive and costly work that requires real-world data sets and filtering, processing and preparing training data. Cosmos uses generative AI to accelerate this process, developing smarter, faster and more accurate AI models for autonomous driving and robots.""","The development of autonomous driving is achieved with three different computers: The NVIDIA DGX system is used in data centers to train AI-based stacks, NVIDIA Omniverse running on NVIDIA OVX systems is used for simulation and synthetic data generation, and the NVIDIA AGX in-car computer processes sensor data in real time to ensure safety. These specialized full-stack systems collectively drive continuous development progress, accelerating efficiency and safety improvements. NVIDIA announced at CES today the addition of a new member to this formula: NVIDIA Cosmos. This platform includes cutting-edge World Foundational Models (WFM), advanced labelers, guardrails, and accelerated video processing pipelines, designed to advance the development of autonomous vehicles and robotic physical AI systems.
+
+By integrating Cosmos into the three-computer solution, developers gain a data flywheel that converts thousands of miles of human driving into billions of miles of virtual driving, improving training data quality. Sanja Fidler, Vice President of NVIDIA AI Research, stated: ""The data flywheel of the autonomous driving data factory includes fleet data collection, precise 4D reconstruction, and AI, to generate scenes and various traffic scenarios for training and closed-loop evaluation. Developers using the NVIDIA Omniverse platform, Cosmos, and supported AI models can generate synthetic driving scenes, amplifying training data manyfold.""
+
+Norm Marks, Vice President of NVIDIA Automotive Products, stated: ""Developing physical AI models has always been a resource-intensive and costly task, requiring real-world datasets, sorting, organizing, and preparing training data. Cosmos accelerates this process utilizing generative AI, making the development of AI models for autonomous driving and robotics smarter, faster, and more accurate.""
+
+Leaders in the transportation field using Cosmos to build physical AI for autonomous driving include:
+- Waabi: Pioneering generative AI for the physical world, using Cosmos to search and organize video data for developing and simulating autonomous driving software.
+- Wayve: Developing AI foundational models for autonomous driving, evaluating Cosmos as a tool for discovering edge and extreme driving scenarios for safety and verification.
+- Autonomous driving toolchain supplier Foretellix uses Cosmos and NVIDIA Omniverse Sensor RTX API to evaluate and produce large volumes of high-fidelity test scenarios and training data.
+
+Additionally, ride-hailing giant Uber will partner with NVIDIA to accelerate the development of autonomous mobility technology. Uber provides rich driving datasets, and combined with the Cosmos platform and NVIDIA DGX Cloud, will help autonomous vehicle partners build stronger AI models more efficiently.
+
+Availability:
+Cosmos WFM is now available under an open model license on Hugging Face and NVIDIA NGC catalog. Cosmos models will soon be offered as fully optimized NVIDIA NIM microservices.
+
+Start using Cosmos, watch demonstrations, and participate in NVIDIA's activities at CES."
+10,0,"A new NVIDIA DRIVE AI Systems Inspection Lab will help automotive ecosystem partners navigate evolving industry standards for autonomous vehicle safety.
+The lab, launched today, will focus on inspecting and verifying that automotive partner software and systems on the
+NVIDIA DRIVE AGX
+platform meet the automotive industry’s stringent safety and cybersecurity standards, including AI functional safety.
+The lab has been accredited by the ANSI National Accreditation Board (
+ANAB
+) according to the ISO/IEC 17020 assessment for standards, including:
+Functional safety (ISO 26262)
+SOTIF (ISO 21448)
+Cybersecurity (ISO 21434)
+UN-R regulations, including UN-R 79, UN-R 13-H, UN-R 152, UN-R 155, UN-R 157 and UN-R 171
+AI functional safety (ISO PAS 8800 and ISO/IEC TR 5469)
+“The launch of this new lab will help partners in the global automotive ecosystem create safe, reliable autonomous driving technology,” said Ali Kani, vice president of automotive at NVIDIA. “With accreditation by ANAB, the lab will carry out an inspection plan that combines functional safety, cybersecurity and AI — bolstering adherence to the industry’s safety standards.”
+“ANAB is proud to be the accreditation body for the NVIDIA DRIVE AI Systems Inspection Lab,” said R. Douglas Leonard Jr., executive director of ANAB. “NVIDIA’s comprehensive evaluation verifies the demonstration of competence and compliance with internationally recognized standards, helping ensure that DRIVE ecosystem partners meet the highest benchmarks for functional safety, cybersecurity and AI integration.”
+The new lab builds on NVIDIA’s ongoing safety compliance work with Mercedes-Benz and JLR. Inaugural participants in the lab include Continental and Sony SSS-America.
+“We are pleased to participate in the newly launched NVIDIA Drive AI Systems Inspection Lab and to further intensify the fruitful, ongoing collaboration between our two companies,” said Nobert Hammerschmidt, head of components business at Continental.
+“Self-driving vehicles have the capability to significantly enhance safety on roads,” said Marius Evensen, head of automotive image sensors at Sony SSS-America. “We look forward to working with NVIDIA’s DRIVE AI Systems Inspection Lab to help us deliver the highest levels of safety to our customers.”","全新啟用的 NVIDIA DRIVE AI 系統檢測實驗室（Systems Inspection Lab）將協助汽車生態系合作夥伴掌握不斷發展的自駕車安全產業標準。
+於今日啟用的這處實驗室，將側重於檢測與驗證汽車合作夥伴在 NVIDIA DRIVE AGX 平台上的軟體與系統，是否符合汽車產業嚴格的安全與資安，包括人工智慧（AI）功能安全。
+該實驗室已獲得美國國家標準協會認可委員會（ANAB）根據 ISO/IEC 17020 評估標準的認證，包括：
+功能安全 （ISO 26262）
+SOTIF （ISO 21448）
+資安 （ISO 21434）
+UN-R 法規，包括 UN-R 79、UN-R 13-H、UN-R 152、UN-R 155、UN-R 157 和 UN-R 171
+AI 功能安全 （ISO PAS 8800 和 ISO/IEC TR 5469）
+NVIDIA 車用產品部門副總裁 Ali Kani 表示：「NVIDIA 成立這個新的實驗室，將協助全球汽車產業生態系的合作夥伴發展出安全可靠的自動駕駛技術。在獲得 ANAB 認證後，實驗室將執行結合功能安全、資安與 AI 的檢測計畫，強化遵守業界安全標準的程度。」
+ANAB 執行董事 R. Douglas Leonard Jr 表示：「ANAB 很榮幸成為 NVIDIA DRIVE AI 系統檢測實驗室的認證機構。NVIDIA 的綜合評估驗證其所展示出的能力與遵守國際公認標準，有助於確保 DRIVE 生態系統合作夥伴遵守功能安全、資安和 AI 整合的最高標準。」
+此全新實驗室建立在 NVIDIA 與 Mercedes-Benz 和 JLR 持續進行的安全合規的基礎上。首批加入該實驗室的業者包括大陸集團（Continental）和 Sony SSS-America。
+大陸集團零組件業務部門負責人 Nobert Hammerschmidt 表示：「我們很高興能加入新成立的 NVIDIA Drive AI 系統檢測實驗室，進一步強化我們雙方一路以來卓越的合作成果。」
+Sony SSS-America 車用影像感測器部門負責人 Marius Evensen 表示：「自駕車能夠大幅提高用路安全。我們期待與 NVIDIA 的 DRIVE AI 系統檢測實驗室合作，協助我們為客戶提供最高等級的安全性。」",NVIDIA 啟用 DRIVE AI 系統檢測實驗室，創下業界全新安全里程碑,https://blogs.nvidia.com.tw/blog/drive-ai-lab-ces/,"NVIDIA today launched its Systems Inspection Lab (SIL) as part of its DRIVE Autonomous System Validation efforts. The lab, which is available today, will help its automotive ecosystem partners navigate the evolving standards for autonomous vehicle safety.
+
+The lab focuses on inspecting and verifying the software and systems of automotive partners on the NVIDIA DRIVE AGX platform to ensure compliance with the rigorous safety and security standards of the automotive industry, including artificial intelligence (AI) functionality safety.
+
+The lab has been accredited by the American National Standards Institute (ANSI)-accredited ANAB (ANSI National Accreditation Board) based on the ISO/IEC 17020 evaluation standard, covering:
+
+Function Safety (ISO 26262)
+SOTIF (ISO 21448)
+Cybersecurity (ISO 21434)
+UN-R regulations, including UN-R 79, UN-R 13-H, UN-R 152, UN-R 155, UN-R 157, and UN-R 171
+AI Function Safety (ISO PAS 8800 and ISO/IEC TR 5469)
+
+""NVIDIA is establishing this new lab to help our global automotive ecosystem partners develop safe and reliable autonomous driving technologies,"" said Ali Kani, vice president and general manager of Automotive at NVIDIA. ""With the ANAB accreditation, the lab will now execute a combined functional safety, cybersecurity, and AI testing plan to further enhance the industry's adherence to safety standards.""
+
+""The ANAB is honored to be the accrediting body for NVIDIA's Systems Inspection Lab,"" said R. Douglas Leonard Jr., ANAB Executive Director. ""NVIDIA's comprehensive evaluation and verification demonstrate its capability and compliance with international recognized standards, helping ensure that DRIVE ecosystem partners adhere to the highest standards of functional safety, cybersecurity and AI integration.""
+
+The new lab is based on the ongoing safety-relevant work of NVIDIA and Mercedes-Benz and JLR. The first customers to join the lab are Continental and Sony SSS-America.
+
+""We are excited to join the newly established NVIDIA DRIVE AI Systems Inspection Lab and further strengthen our longstanding collaborative success,"" said Nobert Hammerschmidt, head of Continental's Components division.
+
+""We believe autonomous driving can greatly enhance road safety,"" said Marius Evensen, director of Automotive Imaging Sensors at Sony SSS-America. ""We look forward to collaborating with NVIDIA's DRIVE AI Systems Inspection Lab to help deliver the highest level of safety for our customers.""","The newly launched NVIDIA DRIVE AI Systems Inspection Lab will assist automotive ecosystem partners in mastering the evolving industry standards for autonomous vehicle safety. This lab, which inaugurated today, will focus on testing and verifying whether the software and systems of automotive partners on the NVIDIA DRIVE AGX platform meet the stringent safety and cybersecurity standards of the automotive industry, including AI functional safety.
+
+The lab is certified by the National Accreditation Board of the United States (ANAB) according to ISO/IEC 17020 evaluation standards, including:
+- Functional Safety (ISO 26262)
+- SOTIF (ISO 21448)
+- Cybersecurity (ISO 21434)
+- UN-R Regulations, including UN-R 79, UN-R 13-H, UN-R 152, UN-R 155, UN-R 157, and UN-R 171
+- AI Functional Safety (ISO PAS 8800 and ISO/IEC TR 5469)
+
+Ali Kani, Vice President of NVIDIA’s Automotive Products Division, stated: ""NVIDIA established this new lab to assist partners within the global automotive industry ecosystem in developing safe and reliable autonomous driving technology. With ANAB certification, the lab will execute testing programs that integrate functional safety, cybersecurity, and AI, enhancing the level of compliance with industry safety standards.""
+
+R. Douglas Leonard Jr, Executive Director of ANAB, stated: ""ANAB is honored to be the certifying body for the NVIDIA DRIVE AI Systems Inspection Lab. NVIDIA’s comprehensive evaluation verifies its demonstrated capabilities and compliance with internationally recognized standards, which helps ensure that the DRIVE ecosystem partners adhere to the highest standards of functional safety, cybersecurity, and AI integration.""
+
+This new lab is built upon NVIDIA's ongoing safety compliance collaboration with Mercedes-Benz and JLR. The initial participants joining the lab include Continental and Sony SSS-America.
+
+Norbert Hammerschmidt, Head of the Component Business Unit at Continental, stated: ""We are delighted to join the newly established NVIDIA DRIVE AI Systems Inspection Lab, further strengthening our already excellent cooperative achievements.""
+
+Marius Evensen, Head of Automotive Image Sensor Division at Sony SSS-America, stated: ""Autonomous vehicles can significantly improve road safety. We look forward to collaborating with NVIDIA's DRIVE AI Systems Inspection Lab to help us provide the highest level of safety for our customers."""
+10,1,"“Compliance with functional safety, SOTIF and cybersecurity is particularly challenging for complex systems such as AI-based autonomous vehicles,” said Riccardo Mariani, head of industry safety at NVIDIA. “Through the DRIVE AI Systems Inspection Lab, the correctness of the integration of our partners’ products with DRIVE safety and cybersecurity requirements can be inspected and verified.”
+Now open to all NVIDIA DRIVE AGX platform partners, the lab is expected to expand to include additional automotive and robotics products and add a testing component.
+Complementing International Automotive Safety Standards
+The NVIDIA DRIVE AI Systems Inspection Lab complements the missions of independent third-party certification bodies, including technical service organizations such as TÜV SÜD, TÜV Rheinland and exida, as well as vehicle certification agencies such as VCA and KBA.
+Today’s announcement dovetails with recent significant safety certifications and assessments of NVIDIA automotive products:
+TÜV SÜD
+granted the ISO 21434 Cybersecurity Process certification to NVIDIA for its automotive system-on-a-chip, platform and software engineering processes. Upon certification release, the
+NVIDIA DriveOS
+6.0 operating system conforms with ISO 26262 Automotive Safety Integrity Level (ASIL) D standards.
+“Meeting cybersecurity process requirements is of fundamental importance in the autonomous vehicle era,” said Martin Webhofer, CEO of TÜV SÜD Rail GmbH. “NVIDIA has successfully established processes, activities and procedures that fulfill the stringent requirements of ISO 21434. Additionally, NVIDIA DriveOS 6.0 conforms to ISO 26262 ASIL D standards, pending final certification activities.”
+TÜV Rheinland
+performed an independent United Nations Economic Commission for Europe safety assessment of NVIDIA DRIVE AV related to safety requirements for complex electronic systems.
+“NVIDIA has demonstrated thorough, high-quality, safety-oriented processes and technologies in the context of the assessment of the generic, non-OEM-specific parts of the SAE level 2 NVIDIA DRIVE system,” said Dominik Strixner, global lead functional safety automotive mobility at TÜV Rheinland.
+To learn more about NVIDIA’s work in advancing autonomous driving safety, read the NVIDIA Self-Driving Safety Report.","NVIDIA 產業安全部門負責人Riccardo Mariani 表示：「對於基於 AI 的自動駕駛車這一類複雜系統來說，遵守功能安全、SOTIF 和資安是一件特別挑戰的事。透過 DRIVE AI 系統檢測實驗室，我們可以檢測和驗證合作夥伴的產品是否有正確與 DRIVE 安全和資安要求進行整合。」
+該實驗室目前開放全體 NVIDIA DRIVE AGX 平台合作夥伴使用，預計將加入更多汽車和機器人產品，並且將增加測試環節。
+與國際汽車安全標準相輔相成
+NVIDIA DRIVE AI 系統檢測實驗室與獨立第三方認證機構的任務相輔相成，包括 TÜV SÜD、TÜV Rheinland 和 exida 等技術服務機構，以及 VCA 和 KBA 等車輛認證機構。
+今天宣布的消息與 NVIDIA 車用產品最近獲得的重要安全認證及評估結果不謀而合：
+TÜV SÜD
+授予 NVIDIA 汽車系統單晶片、平台與軟體工程流程 ISO 21434 網路安全流程認證。認證發布後，
+NVIDIA DriveOS
+6.0 作業系統符合 ISO 26262 車輛安全完整性等級（ASIL）D 標準。
+TÜV SÜD Rail GmbH 執行長 Martin Webhofer 表示：「在自動駕駛時代，必須要符合網路安全流程的要求。NVIDIA 已經成功建立了符合 ISO 21434 嚴格要求的流程、活動和程序。通過最後的認證活動後，NVIDIA DriveOS 6.0 還符合 ISO 26262 ASIL D 標準。」
+TÜV Rheinland
+對 NVIDIA DRIVE AV 進行了與複雜電子系統安全要求相關的聯合國歐洲經濟委員會獨立安全評估作業。
+TÜV Rheinland 全球功能安全汽車行動部門主管 Dominik Strixner 表示：「NVIDIA 在 SAE level 2 NVIDIA DRIVE 系統通用、非 OEM 專用零件的評估中，展現出全面、高品質、以安全為導向的流程與技術。」
+若要進一步瞭解 NVIDIA 在推動自動駕駛安全方面的各項工作，請閱讀 NVIDIA 自動駕駛安全報告。",NVIDIA 啟用 DRIVE AI 系統檢測實驗室，創下業界全新安全里程碑,https://blogs.nvidia.com.tw/blog/drive-ai-lab-ces/,"“Given the complexity of this type of system, it's especially challenging to meet the requirements for functional safety, SOTIF and cybersecurity,"" said Riccardo Mariani, Head of Industry Security at NVIDIA. ""Through the DRIVE AI Systems Test Lab, we can test and validate our partners' products to ensure they are properly integrated with DRIVE security and cybersecurity requirements.""
+
+The lab is currently available to all NVIDIA DRIVE AGX platform partners and will be expanded to include additional automotive and robotics products with more testing segments.
+
+The NVIDIA DRIVE AI Systems Test Lab complements the work of independent third-party certification bodies, including TÜV SÜD, TÜV Rheinland and exida, as well as VCA and KBA, automotive certification bodies.
+
+The announcement aligns with recent important security certification and evaluation results for NVIDIA's automotive products:
+
+TÜV SÜD
+
+* Granted NVIDIA automotive system-on-chip, platform and software engineering process ISO 21434 network security process certification.
+* Following the publication of the certification, NVIDIA DriveOS 6.0 meets ISO 26262 vehicle safety integrity level (ASIL) D standard.
+NVIDIA DriveOS 6.0 meets ISO 26262 vehicle safety integrity level (ASIL) D standard.
+
+TÜV SÜD Rail GmbH CEO Martin Webhofer said: “In the era of autonomous driving, it is necessary to meet the requirements for network security processes. NVIDIA has successfully established a process, activities and procedures that meet the strict requirements of ISO 21434. Following the final certification activities, NVIDIA DriveOS 6.0 meets ISO 26262 ASIL D standard.”
+
+TÜV Rheinland
+
+* Conducted an independent safety assessment, related to the requirements for security of complex electronic systems, in accordance with United Nations Economic Commission for Europe.
+Dominik Strixner, Global Functional Safety Automotive Business Unit Manager at TÜV Rheinland, said: “NVIDIA showed a comprehensive, high-quality and safety-oriented process and technology in the evaluation of its SAE level 2, general-use, non-OEM-specific components of the NVIDIA DRIVE system.""
+
+To learn more about NVIDIA’s work in advancing the safety of autonomous driving, please read the NVIDIA Autonomous Driving Safety Report.","Riccardo Mariani, head of NVIDIA's industrial safety department, stated: ""For complex systems such as AI-based autonomous vehicles, complying with functional safety, SOTIF, and cybersecurity is a particularly challenging task. Through the DRIVE AI Systems Testing Lab, we can test and verify whether our partners' products are properly integrated with DRIVE safety and cybersecurity requirements.""
+
+The lab is currently open to all NVIDIA DRIVE AGX platform partners and plans to include more automotive and robotic products, as well as increase the testing scenarios.
+
+Complementing International Automotive Safety Standards  
+The NVIDIA DRIVE AI Systems Testing Lab complements the work of independent third-party certification bodies, including technical service organizations like TÜV SÜD, TÜV Rheinland, and exida, as well as vehicle certification bodies like VCA and KBA.
+
+Today's announcement coincides with recent significant safety certifications and evaluation results for NVIDIA automotive products:
+
+TÜV SÜD  
+Granted the certification for ISO 21434 cybersecurity process for NVIDIA's automotive system-on-chips, platforms, and software engineering processes. After the certification was issued, NVIDIA DriveOS 6.0 complies with the ISO 26262 Automotive Safety Integrity Level (ASIL) D standard.  
+Martin Webhofer, CEO of TÜV SÜD Rail GmbH, stated: ""In the era of autonomous driving, meeting cybersecurity process requirements is essential. NVIDIA has successfully established processes, activities, and procedures that meet the stringent requirements of ISO 21434. After the final certification activities, NVIDIA DriveOS 6.0 also complies with the ISO 26262 ASIL D standard.""
+
+TÜV Rheinland  
+Conducted an independent safety assessment operation related to the safety requirements of complex electronic systems for NVIDIA DRIVE AV as part of the United Nations Economic Commission for Europe.  
+Dominik Strixner, Global Head of Functional Safety Automotive at TÜV Rheinland, said: ""NVIDIA demonstrated comprehensive, high-quality, safety-oriented processes and technologies in the evaluation of SAE Level 2 NVIDIA DRIVE systems, which are generic, non-OEM specific components.""
+
+To learn more about NVIDIA's efforts in promoting autonomous driving safety, please read the NVIDIA Autonomous Driving Safety Report."

vecalign/vecalign.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import argparse
+import logging
+import pickle
+from math import ceil
+from random import seed as seed
+
+import numpy as np
+
+logger = logging.getLogger('vecalign')
+logger.setLevel(logging.WARNING)
+logFormatter = logging.Formatter("%(asctime)s  %(levelname)-5.5s  %(message)s")
+consoleHandler = logging.StreamHandler()
+consoleHandler.setFormatter(logFormatter)
+logger.addHandler(consoleHandler)
+
+from dp_utils import make_alignment_types, make_one_to_many_alignment_types, print_alignments, read_alignments, \
+    read_in_embeddings, make_doc_embedding, vecalign
+
+from score import score_multiple, log_final_scores
+
+
+def _main():
+    # make runs consistent
+    seed(42)
+    np.random.seed(42)
+
+    parser = argparse.ArgumentParser('Sentence alignment using sentence embeddings and FastDTW',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('-s', '--src', type=str, nargs='+', required=True,
+                        help='preprocessed source file to align')
+
+    parser.add_argument('-t', '--tgt', type=str, nargs='+', required=True,
+                        help='preprocessed target file to align')
+
+    parser.add_argument('-g', '--gold_alignment', type=str, nargs='+', required=False,
+                        help='preprocessed target file to align')
+
+    parser.add_argument('--src_embed', type=str, nargs=2, required=True,
+                        help='Source embeddings. Requires two arguments: first is a text file, sencond is a binary embeddings file. ')
+
+    parser.add_argument('--tgt_embed', type=str, nargs=2, required=True,
+                        help='Target embeddings. Requires two arguments: first is a text file, sencond is a binary embeddings file. ')
+
+    parser.add_argument('-a', '--alignment_max_size', type=int, default=4,
+                        help='Searches for alignments up to size N-M, where N+M <= this value. Note that the the embeddings must support the requested number of overlaps')
+
+    # without flag: one_to_many==default, with flag but no argument: one_to_many==const, with flag and argument: one_to_many==argument
+    parser.add_argument('--one_to_many', type=int, nargs='?', default=None, const=50,
+                        help='Perform one to many (e.g. 1:1, 1:2, ... 1:M) alignment.'
+                        ' Argument specifies M but will default to 50 if flag is set but no argument is provided. Overrides --alignment_max_size (-a).')
+
+    parser.add_argument('-d', '--del_percentile_frac', type=float, default=0.2,
+                        help='Deletion penalty is set to this percentile (as a fraction) of the cost matrix distribution. Should be between 0 and 1.')
+
+    parser.add_argument('-v', '--verbose', help='sets consle to logging.DEBUG instead of logging.WARN',
+                        action='store_true')
+
+    parser.add_argument('--max_size_full_dp', type=int, default=300,
+                        help='Maximum size N for which is is acceptable to run full N^2 dynamic programming.')
+
+    parser.add_argument('--costs_sample_size', type=int, default=20000,
+                        help='Sample size to estimate costs distribution, used to set deletion penalty in conjunction with deletion_percentile.')
+
+    parser.add_argument('--num_samps_for_norm', type=int, default=100,
+                        help='Number of samples used for normalizing embeddings')
+
+    parser.add_argument('--search_buffer_size', type=int, default=5,
+                        help='Width (one side) of search buffer. Larger values makes search more likely to recover from errors but increases runtime.')
+
+    parser.add_argument('--debug_save_stack', type=str,
+                        help='Write stack to pickle file for debug purposes')
+
+    parser.add_argument('--print_aligned_text', action='store_true',
+                        help='Print aligned text in addition to alignments, for debugging/tuning.')
+
+    args = parser.parse_args()
+
+    if len(args.src) != len(args.tgt):
+        raise Exception('number of source files must match number of target files')
+
+    if args.gold_alignment is not None:
+        if len(args.gold_alignment) != len(args.src):
+            raise Exception('number of gold alignment files, if provided, must match number of source and target files')
+
+    if args.verbose:
+        import logging
+        logger.setLevel(logging.INFO)
+
+    if args.alignment_max_size < 2:
+        logger.warning('Alignment_max_size < 2. Increasing to 2 so that 1-1 alignments will be considered')
+        args.alignment_max_size = 2
+
+    src_sent2line, src_line_embeddings = read_in_embeddings(args.src_embed[0], args.src_embed[1])
+    tgt_sent2line, tgt_line_embeddings = read_in_embeddings(args.tgt_embed[0], args.tgt_embed[1])
+
+    src_max_alignment_size = 1 if args.one_to_many is not None else args.alignment_max_size
+    tgt_max_alignment_size = args.one_to_many if args.one_to_many is not None else args.alignment_max_size
+
+    width_over2 = ceil(max(src_max_alignment_size, tgt_max_alignment_size) / 2.0) + args.search_buffer_size
+
+    test_alignments = []
+    stack_list = []
+    for src_file, tgt_file in zip(args.src, args.tgt):
+        logger.info('Aligning src="%s" to tgt="%s"', src_file, tgt_file)
+
+        src_lines = open(src_file, 'rt', encoding="utf-8").readlines()
+        vecs0 = make_doc_embedding(src_sent2line, src_line_embeddings, src_lines, src_max_alignment_size)
+
+        tgt_lines = open(tgt_file, 'rt', encoding="utf-8").readlines()
+        vecs1 = make_doc_embedding(tgt_sent2line, tgt_line_embeddings, tgt_lines, tgt_max_alignment_size)
+
+        if args.one_to_many is not None:
+            final_alignment_types = make_one_to_many_alignment_types(args.one_to_many)
+        else:
+            final_alignment_types = make_alignment_types(args.alignment_max_size)
+        logger.debug('Considering alignment types %s', final_alignment_types)
+
+        stack = vecalign(vecs0=vecs0,
+                         vecs1=vecs1,
+                         final_alignment_types=final_alignment_types,
+                         del_percentile_frac=args.del_percentile_frac,
+                         width_over2=width_over2,
+                         max_size_full_dp=args.max_size_full_dp,
+                         costs_sample_size=args.costs_sample_size,
+                         num_samps_for_norm=args.num_samps_for_norm)
+
+        # write final alignments to stdout
+        print_alignments(stack[0]['final_alignments'], scores=stack[0]['alignment_scores'],
+                         src_lines=src_lines if args.print_aligned_text else None,
+                         tgt_lines=tgt_lines if args.print_aligned_text else None)
+
+        test_alignments.append(stack[0]['final_alignments'])
+        stack_list.append(stack)
+
+    if args.gold_alignment is not None:
+        gold_list = [read_alignments(x) for x in args.gold_alignment]
+        res = score_multiple(gold_list=gold_list, test_list=test_alignments)
+        log_final_scores(res)
+
+    if args.debug_save_stack:
+        pickle.dump(stack_list, open(args.debug_save_stack, 'wb'))
+
+
+if __name__ == '__main__':
+    _main()