diff --git "a/competition/13_Qwen2_7b_finetuning_l40.ipynb" "b/competition/13_Qwen2_7b_finetuning_l40.ipynb" new file mode 100644--- /dev/null +++ "b/competition/13_Qwen2_7b_finetuning_l40.ipynb" @@ -0,0 +1,6455 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "mo-H82fsy1jy" + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "bYpMYw2Wz7Bv", + "outputId": "cc9dce0a-c5f4-421c-ed41-d79c3a1b3577" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "workding dir: c:\\Users\\HT\\Documents\\URP\\logical-reasoning\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "workding_dir = str(Path.cwd().parent)\n", + "os.chdir(workding_dir)\n", + "sys.path.append(workding_dir)\n", + "print(\"working dir:\", workding_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aA2yLesz27M8", + "outputId": "32909874-deee-44b8-c3de-5476cc3008f1" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "need_to_setup_env = False\n", + "need_to_setup_env" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "u0QXyHU_5DQR", + "outputId": "54672b45-b5dc-48ef-efd2-5e8545e7b78b" + }, + "outputs": [], + "source": [ + "if need_to_setup_env:\n", + " %pip install -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RKmGaYU_5OkA", + "outputId": "27c8c14b-1538-41e0-e3dd-dc37c650f5fd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loading env vars from: c:\\Users\\HT\\Documents\\URP\\logical-reasoning\\.env.qwen2_7b\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dotenv import find_dotenv, load_dotenv\n", + "\n", + "found_dotenv = find_dotenv(\".env.qwen2_7b\")\n", + "\n", + "if len(found_dotenv) == 0:\n", + " found_dotenv = find_dotenv(\".env.example\")\n", + "print(f\"loading env vars from: {found_dotenv}\")\n", + "load_dotenv(found_dotenv, override=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Xa7KxkuzUeS9", + "outputId": "6c71b30e-7b02-44ef-feeb-df94989be7f3" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Qwen/Qwen2-7B None False datasets/mgtv results/mgtv-results_qwen2_7b.csv False\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "model_name = os.getenv(\"MODEL_NAME\")\n", + "adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n", + "load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n", + "data_path = os.getenv(\"LOGICAL_REASONING_DATA_PATH\")\n", + "results_path = os.getenv(\"LOGICAL_REASONING_RESULTS_PATH\")\n", + "use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n", + "\n", + "print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path, use_english_datasets)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 379 + }, + "id": "goEFOG9Z5TvW", + "outputId": "1491df15-1eca-43ac-89d2-ca1f74b42297" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
textlabelanswertitlepuzzletruth
0偷的人信神吗不是NaN乡村之谜：消失的南瓜在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民...真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...
1偷南瓜是为了来年丰收吗不是NaN乡村之谜：消失的南瓜在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民...真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...
2村庄里的人喜欢南瓜嘛不重要NaN乡村之谜：消失的南瓜在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民...真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...
3村庄里的人每年都需要用南瓜做祭品嘛不是NaN乡村之谜：消失的南瓜在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民...真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...
4是村里的人偷的么是NaN乡村之谜：消失的南瓜在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民...真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季...
\n", + "
" + ], + "text/plain": [ + " text label answer title \\\n", + "0 偷的人信神吗 不是 NaN 乡村之谜：消失的南瓜 \n", + "1 偷南瓜是为了来年丰收吗 不是 NaN 乡村之谜：消失的南瓜 \n", + "2 村庄里的人喜欢南瓜嘛 不重要 NaN 乡村之谜：消失的南瓜 \n", + "3 村庄里的人每年都需要用南瓜做祭品嘛 不是 NaN 乡村之谜：消失的南瓜 \n", + "4 是村里的人偷的么 是 NaN 乡村之谜：消失的南瓜 \n", + "\n", + " puzzle \\\n", + "0 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民... \n", + "1 在甄家村里���有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民... \n", + "2 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民... \n", + "3 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民... \n", + "4 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民... \n", + "\n", + " truth \n", + "0 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季... \n", + "1 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季... \n", + "2 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季... \n", + "3 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季... \n", + "4 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季... " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "df = pd.read_csv(\"datasets/mgtv/train.csv\")\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zPuVZfdRICtY", + "outputId": "7b423111-fe0e-47ab-eeb8-438bc10c7930" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'instruction': '你是一个逻辑游戏的主持人。游戏规则如下：1. 参与者会得到一个谜题。2. 参与者可以通过提问来获取线索，尝试解开谜题。3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。请严格按照这些规则回答参与者提出的问题。谜题: {}实际情况: {}参与者提出的问题: {}',\n", + " 'input': '谜题: 乡村之谜：消失的南瓜 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民们对此现象困惑不解。请找出南瓜失踪背后的原因。实际情况: 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季节结婚。然而，命运弄人，姑娘在婚礼前的一场意外中离世。悲伤的农夫为了纪念心爱的姑娘，每年都会将最大的南瓜偷走，放到姑娘的墓前，以此寄托自己的哀思。这一行为延续了多年，成为了乡村里一个神秘的传说。参与者提出的问题: 偷的人信神吗',\n", + " 'output': '不是'}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset_data = [\n", + " {\n", + " \"instruction\": \"你是一个逻辑游戏的主持人。游戏规则如下：1. 参与者会得到一个谜题。2. 参与者可以通过提问来获取线索，尝试解开谜题。3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。请严格按照这些规则回答参与者提出的问题。谜题: {}实际情况: {}参与者提出的问题: {}\",\n", + " \"input\": \"谜题: \" + row_dict[\"title\"] + \" \" + row_dict[\"puzzle\"] + \"实际情况: \" + row_dict[\"truth\"] + \"参与者提出的问题: \" + row_dict[\"text\"],\n", + " \"output\": row_dict[\"label\"]\n", + " }\n", + " for row_dict in df.to_dict(orient=\"records\")\n", + "]\n", + "\n", + "dataset_data[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "unuNtJc5_AIL", + "outputId": "d3b87976-e32d-4b8f-fcc9-56b048604526" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "JSON file saved to /content/LLaMA-Factory/data/mgtv_train.json\n" + ] + } + ], + "source": [ + "import os\n", + "import json\n", + "\n", + "# Define the directory where you want to save the JSON file\n", + "output_dir = \"/content/LLaMA-Factory/data/\"\n", + "\n", + "# Ensure the directory exists\n", + "os.makedirs(output_dir, exist_ok=True)\n", + "\n", + "# Define the full path for the JSON file\n", + "json_file_path = os.path.join(output_dir, \"mgtv_train.json\")\n", + "\n", + "# Save the dataset data to the specified path\n", + "with open(json_file_path, \"w\") as f:\n", + " json.dump(dataset_data, f)\n", + "\n", + "print(f\"JSON file saved to {json_file_path}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zyxvE1nfX8cq", + "outputId": "1d3bddc5-289f-48b7-c2ce-5e9bd1684ea0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/content/LLaMA-Factory\n" + ] + } + ], + "source": [ + "import json\n", + "%cd /content/LLaMA-Factory/\n", + "\n", + "args = dict(\n", + " model_name_or_path=\"Qwen/Qwen2-7B\", # use Qwen/Qwen2-7B-Instruct model\n", + "\n", + " stage=\"sft\", # do supervised fine-tuning\n", + " do_train=True,\n", + " finetuning_type=\"lora\", # use LoRA adapters to save memory\n", + " lora_target=\"all\", # attach LoRA adapters to all linear layers\n", + " quantization_bit=4,\n", + " loraplus_lr_ratio=16.0, # 16x base LoRA learning rate\n", + "\n", + " dataset=\"mgtv_train\",\n", + " template=\"qwen\",\n", + " cutoff_len=4096,\n", + " max_samples=5000,\n", + " overwrite_cache=\"true\",\n", + " preprocessing_num_workers=16,\n", + "\n", + " output_dir=\"/content/qwen2-7b\",\n", + " logging_steps=562,\n", + " save_steps=562,\n", + " plot_loss=\"true\",\n", + " overwrite_output_dir=\"true\",\n", + "\n", + " per_device_train_batch_size=1, # the batch size\n", + " gradient_accumulation_steps=8, # the gradient accumulation steps\n", + " learning_rate=0.001, # the learning rate\n", + " num_train_epochs=6.0, # the epochs of training\n", + " lr_scheduler_type=\"cosine\", # use cosine learning rate scheduler\n", + " warmup_ratio=0.1, # use warmup scheduler\n", + " bf16=True,\n", + " ddp_timeout=180000000, #5.71 years lol\n", + "\n", + " val_size=0.1,\n", + " per_device_eval_batch_size=1,\n", + " eval_strategy=\"steps\",\n", + " eval_steps=562,\n", + "\n", + " report_to=\"wandb\",\n", + ")\n", + "\n", + "with open(\"train_qwen2_7b.json\", \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(args, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QlYqm4TePib3" + }, + "outputs": [], + "source": [ + "with open(\"data/dataset_info.json\", 'r+') as file:\n", + " # First we load existing data into a dict.\n", + " file_data = json.load(file)\n", + " # Insert new_data at the beginning of the emp_details list.\n", + " qwen2_7b = {\"mgtv_train\": {\n", + " \"file_name\": \"mgtv_train.json\"\n", + " }\n", + " }\n", + "\n", + " qwen2_7b.update(file_data)\n", + " file.seek(0)\n", + " # convert back to json.\n", + " json.dump(qwen2_7b, file, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "background_save": true, + "base_uri": "https://localhost:8080/" + }, + "id": "VEuCMjMpITg-", + "outputId": "76cf7882-3ae8-4c53-8d6c-c59b3557af0e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-07-15 14:34:28.658348: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2024-07-15 14:34:28.710574: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-07-15 14:34:28.710630: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-07-15 14:34:28.712064: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-07-15 14:34:28.719927: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-15 14:34:29.954969: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "07/15/2024 14:34:36 - WARNING - llamafactory.hparams.parser - We recommend enable `upcast_layernorm` in quantized training.\n", + "07/15/2024 14:34:36 - INFO - llamafactory.hparams.parser - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16\n", + "tokenizer_config.json: 100% 1.29k/1.29k [00:00<00:00, 9.74MB/s]\n", + "vocab.json: 100% 2.78M/2.78M [00:00<00:00, 10.4MB/s]\n", + "merges.txt: 100% 1.67M/1.67M [00:00<00:00, 6.67MB/s]\n", + "tokenizer.json: 100% 7.03M/7.03M [00:00<00:00, 18.8MB/s]\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/vocab.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/merges.txt\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-15 14:34:38,471 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer_config.json\n", + "[WARNING|logging.py:314] 2024-07-15 14:34:38,733 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/15/2024 14:34:38 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/15/2024 14:34:38 - INFO - llamafactory.data.loader - Loading dataset mgtv_train.json...\n", + "Generating train split: 25000 examples [00:01, 18396.69 examples/s]\n", + "/usr/local/lib/python3.10/dist-packages/multiprocess/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n", + " self.pid = os.fork()\n", + "Converting format of dataset (num_proc=16): 100% 5000/5000 [00:00<00:00, 21217.02 examples/s]\n", + "Running tokenizer on dataset (num_proc=16): 100% 5000/5000 [00:02<00:00, 1705.27 examples/s]\n", + "training example:\n", + "input_ids:\n", + "[151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 56568, 101909, 104913, 99329, 9370, 106040, 1773, 99329, 104190, 104506, 5122, 16, 13, 26853, 224, 57218, 28946, 36993, 101051, 46944, 107969, 33872, 1773, 17, 13, 26853, 224, 57218, 28946, 105125, 107666, 36407, 45912, 105814, 3837, 104482, 117647, 107969, 33872, 1773, 18, 13, 69162, 34204, 103991, 86119, 3837, 106040, 44063, 100345, 107591, 102104, 87752, 105220, 109487, 100653, 5122, 20412, 5373, 99520, 5373, 16530, 99335, 5373, 102104, 88991, 5373, 56007, 24339, 32100, 1773, 19, 13, 49602, 252, 99590, 15946, 53153, 42855, 99885, 102158, 27369, 3837, 105827, 65770, 99475, 109487, 101047, 110281, 18600, 1773, 77557, 3837, 108620, 99360, 2073, 99520, 854, 65770, 99475, 12857, 2073, 16530, 55807, 20, 13, 26853, 224, 57218, 28946, 85106, 100345, 102104, 36407, 113272, 90395, 103941, 109363, 107969, 33872, 9370, 88991, 102349, 1773, 14880, 110439, 100001, 104190, 102104, 111842, 101080, 103936, 1773, 107969, 33872, 25, 4687, 107591, 25, 4687, 111842, 101080, 103936, 25, 5613, 107969, 33872, 25, 220, 100833, 53930, 107969, 5122, 102505, 9370, 115865, 73562, 109628, 45629, 105489, 3837, 104133, 111718, 106023, 5122, 101988, 115865, 110731, 9370, 105419, 3837, 115865, 99810, 69249, 59743, 104133, 104003, 115865, 36993, 16530, 101401, 68536, 99723, 3837, 115967, 104270, 102060, 110666, 112031, 1773, 14880, 109363, 115865, 110786, 101423, 104249, 1773, 107591, 25, 10236, 250, 253, 48921, 101221, 57218, 101961, 7948, 100894, 9370, 99288, 99818, 101063, 1773, 104269, 99288, 99818, 100774, 13343, 3837, 99798, 57218, 101961, 105664, 102373, 48921, 100271, 1773, 99650, 105616, 18493, 115865, 110731, 9370, 105419, 104388, 1773, 103968, 3837, 102606, 102115, 17340, 3837, 102373, 18493, 106340, 24562, 99774, 82224, 104424, 15946, 99372, 99244, 1773, 110597, 9370, 99288, 99818, 100012, 101416, 63109, 99242, 9370, 102373, 3837, 101988, 101938, 44063, 104003, 115865, 101329, 99314, 3837, 107974, 102373, 9370, 104575, 24562, 3837, 105699, 116418, 100005, 103000, 90663, 1773, 100147, 101070, 105443, 34187, 100097, 3837, 104989, 100833, 69249, 46944, 105190, 9370, 106023, 1773, 111842, 101080, 103936, 25, 4891, 223, 115, 100623, 21317, 99315, 101037, 151645, 198, 151644, 77091, 198, 99520, 151645]\n", + "inputs:\n", + "<|im_start|>system\n", + "You are a helpful assistant.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：1. 参与者会得到一个谜题。2. 参与者可以通过提问来获取线索，尝试解开谜题。3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。请严格按照这些规则回答参与者提出的问题。谜题: {}实际情况: {}参与者提出的问题: {}\n", + "谜题: 乡村之谜：消失的南瓜 在甄家村里，有一个古老的传说：每年南瓜丰收的季节，南瓜田里总有一个最大的南瓜会不翼而飞，村民们对此现象困惑不解。请找出南瓜失踪背后的原因。实际情况: 真相原来与一位年迈的农夫有关。这位农夫年轻时，曾与一位美丽的姑娘相恋。他们约定在南瓜丰收的季节结婚。然而，命运弄人，姑娘在婚礼前的一场意外中离世。悲伤的农夫为了纪念心爱的姑娘，每年都会将最大的南瓜偷走，放到姑娘的墓前，以此寄托自己的哀思。这一行为延续了多年，成为了乡村里一个神秘的传说。参与者提出的问题: 偷的人信神吗<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "label_ids:\n", + "[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 99520, 151645]\n", + "labels:\n", + "不是<|im_end|>\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "config.json: 100% 664/664 [00:00<00:00, 5.23MB/s]\n", + "[INFO|configuration_utils.py:733] 2024-07-15 14:34:44,448 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 14:34:44,451 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "07/15/2024 14:34:44 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit with bitsandbytes.\n", + "model.safetensors.index.json: 100% 27.8k/27.8k [00:00<00:00, 99.4MB/s]\n", + "[INFO|modeling_utils.py:3474] 2024-07-15 14:34:44,977 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/model.safetensors.index.json\n", + "Downloading shards: 0% 0/4 [00:00> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:962] 2024-07-15 14:36:16,412 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643\n", + "}\n", + "\n", + "Loading checkpoint shards: 100% 4/4 [00:06<00:00, 1.55s/it]\n", + "[INFO|modeling_utils.py:4280] 2024-07-15 14:36:26,291 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4288] 2024-07-15 14:36:26,291 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-7B.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "generation_config.json: 100% 138/138 [00:00<00:00, 1.11MB/s]\n", + "[INFO|configuration_utils.py:917] 2024-07-15 14:36:26,489 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/generation_config.json\n", + "[INFO|configuration_utils.py:962] 2024-07-15 14:36:26,489 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"max_new_tokens\": 2048\n", + "}\n", + "\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.model_utils.misc - Found linear modules: q_proj,down_proj,o_proj,gate_proj,v_proj,up_proj,k_proj\n", + "07/15/2024 14:36:27 - INFO - llamafactory.model.loader - trainable params: 20,185,088 || all params: 7,635,801,600 || trainable%: 0.2643\n", + "[INFO|trainer.py:641] 2024-07-15 14:36:27,732 >> Using auto half precision backend\n", + "07/15/2024 14:36:28 - INFO - llamafactory.train.trainer_utils - Using LoRA+ optimizer with loraplus lr ratio 16.00.\n", + "[INFO|trainer.py:2078] 2024-07-15 14:36:28,977 >> ***** Running training *****\n", + "[INFO|trainer.py:2079] 2024-07-15 14:36:28,977 >> Num examples = 4,500\n", + "[INFO|trainer.py:2080] 2024-07-15 14:36:28,977 >> Num Epochs = 6\n", + "[INFO|trainer.py:2081] 2024-07-15 14:36:28,977 >> Instantaneous batch size per device = 1\n", + "[INFO|trainer.py:2084] 2024-07-15 14:36:28,977 >> Total train batch size (w. parallel, distributed & accumulation) = 8\n", + "[INFO|trainer.py:2085] 2024-07-15 14:36:28,977 >> Gradient Accumulation steps = 8\n", + "[INFO|trainer.py:2086] 2024-07-15 14:36:28,977 >> Total optimization steps = 3,372\n", + "[INFO|trainer.py:2087] 2024-07-15 14:36:28,981 >> Number of trainable parameters = 20,185,088\n", + "[INFO|integration_utils.py:723] 2024-07-15 14:36:28,986 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33minflaton-sg\u001b[0m (\u001b[33minflaton-ai\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.4\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/LLaMA-Factory/wandb/run-20240715_143630-ancw8jgs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m/content/qwen2-7b\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/ancw8jgs\u001b[0m\n", + "{'loss': 1.9143, 'grad_norm': 2.1186106204986572, 'learning_rate': 0.000986610734407955, 'epoch': 1.0}\n", + " 17% 562/3372 [1:01:09<5:02:54, 6.47s/it][INFO|trainer.py:3719] 2024-07-15 15:37:39,784 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 15:37:39,784 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 15:37:39,785 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-562\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 15:39:50,101 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 15:39:50,102 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 15:39:50,298 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-562/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 15:39:50,298 >> Special tokens file saved in /content/qwen2-7b/checkpoint-562/special_tokens_map.json\n", + "{'loss': 0.847, 'grad_norm': 0.33948227763175964, 'learning_rate': 0.0008433439152121052, 'epoch': 2.0}\n", + " 33% 1124/3372 [2:04:27<4:05:43, 6.56s/it][INFO|trainer.py:3719] 2024-07-15 16:40:58,115 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 16:40:58,115 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 16:40:58,115 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-1124\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 16:43:11,686 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 16:43:11,687 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 16:43:11,872 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-1124/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 16:43:11,873 >> Special tokens file saved in /content/qwen2-7b/checkpoint-1124/special_tokens_map.json\n", + "{'loss': 0.5831, 'grad_norm': 0.08739642798900604, 'learning_rate': 0.0005870506865895984, 'epoch': 3.0}\n", + " 50% 1686/3372 [3:08:20<3:02:02, 6.48s/it][INFO|trainer.py:3719] 2024-07-15 17:44:50,834 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 17:44:50,834 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 17:44:50,835 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-1686\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 17:47:04,904 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 17:47:04,905 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 17:47:05,097 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-1686/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 17:47:05,098 >> Special tokens file saved in /content/qwen2-7b/checkpoint-1686/special_tokens_map.json\n", + "{'loss': 0.5469, 'grad_norm': 0.5723391771316528, 'learning_rate': 0.00030210098232438424, 'epoch': 4.0}\n", + " 67% 2248/3372 [4:12:15<2:04:11, 6.63s/it][INFO|trainer.py:3719] 2024-07-15 18:48:45,813 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 18:48:45,813 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 18:48:45,814 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-2248\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 18:50:59,783 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 18:50:59,784 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 18:50:59,970 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-2248/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 18:50:59,970 >> Special tokens file saved in /content/qwen2-7b/checkpoint-2248/special_tokens_map.json\n", + "{'loss': 0.5323, 'grad_norm': 0.26129332184791565, 'learning_rate': 8.229824704832284e-05, 'epoch': 5.0}\n", + " 83% 2810/3372 [5:16:10<1:01:38, 6.58s/it][INFO|trainer.py:3719] 2024-07-15 19:52:41,540 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 19:52:41,541 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 19:52:41,541 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-2810\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 19:54:55,909 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 19:54:55,910 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 19:54:56,097 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-2810/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 19:54:56,098 >> Special tokens file saved in /content/qwen2-7b/checkpoint-2810/special_tokens_map.json\n", + "{'loss': 0.5229, 'grad_norm': 0.18251831829547882, 'learning_rate': 0.0, 'epoch': 5.99}\n", + "100% 3372/3372 [6:20:08<00:00, 6.71s/it][INFO|trainer.py:3719] 2024-07-15 20:56:39,578 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 20:56:39,578 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 20:56:39,578 >> Batch size = 1\n", + "\n", + " 0% 0/500 [00:00> Saving model checkpoint to /content/qwen2-7b/checkpoint-3372\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 20:58:53,452 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 20:58:53,453 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 20:58:53,632 >> tokenizer config file saved in /content/qwen2-7b/checkpoint-3372/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 20:58:53,633 >> Special tokens file saved in /content/qwen2-7b/checkpoint-3372/special_tokens_map.json\n", + "[INFO|trainer.py:2329] 2024-07-15 20:58:54,110 >> \n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n", + "{'train_runtime': 22945.1289, 'train_samples_per_second': 1.177, 'train_steps_per_second': 0.147, 'train_loss': 0.8244021631786125, 'epoch': 5.99}\n", + "100% 3372/3372 [6:22:23<00:00, 6.80s/it]\n", + "[INFO|trainer.py:3410] 2024-07-15 20:58:54,115 >> Saving model checkpoint to /content/qwen2-7b\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-15 20:58:54,406 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-15 20:58:54,407 >> Model config Qwen2Config {\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "[INFO|tokenization_utils_base.py:2513] 2024-07-15 20:58:54,600 >> tokenizer config file saved in /content/qwen2-7b/tokenizer_config.json\n", + "[INFO|tokenization_utils_base.py:2522] 2024-07-15 20:58:54,600 >> Special tokens file saved in /content/qwen2-7b/special_tokens_map.json\n", + "***** train metrics *****\n", + " epoch = 5.9947\n", + " total_flos = 396658758GF\n", + " train_loss = 0.8244\n", + " train_runtime = 6:22:25.12\n", + " train_samples_per_second = 1.177\n", + " train_steps_per_second = 0.147\n", + "Figure saved at: /content/qwen2-7b/training_loss.png\n", + "Figure saved at: /content/qwen2-7b/training_eval_loss.png\n", + "Figure saved at: /content/qwen2-7b/training_eval_accuracy.png\n", + "[INFO|trainer.py:3719] 2024-07-15 20:58:55,263 >> ***** Running Evaluation *****\n", + "[INFO|trainer.py:3721] 2024-07-15 20:58:55,264 >> Num examples = 500\n", + "[INFO|trainer.py:3724] 2024-07-15 20:58:55,264 >> Batch size = 1\n", + "100% 500/500 [02:13<00:00, 3.74it/s]\n", + "***** eval metrics *****\n", + " epoch = 5.9947\n", + " eval_accuracy = 0.7747\n", + " eval_loss = 0.513\n", + " eval_runtime = 0:02:13.97\n", + " eval_samples_per_second = 3.732\n", + " eval_steps_per_second = 3.732\n", + "[INFO|modelcard.py:450] 2024-07-15 21:01:09,246 >> Dropping the following result as it does not have all the necessary fields:\n", + "{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}, 'metrics': [{'name': 'Accuracy', 'type': 'accuracy', 'value': 0.7746666666666665}]}\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/accuracy ▁▅█████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss █▄▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime ▁▇█▇█▇█\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second █▂▁▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second █▂▁▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch ▁▁▂▂▄▄▅▅▇▇████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step ▁▁▂▂▄▄▅▅▇▇████\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm █▂▁▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate █▇▅▃▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss █▃▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/accuracy 0.77467\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/loss 0.51301\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/runtime 133.9784\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 3.732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: eval/steps_per_second 3.732\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: total_flos 4.259090989881262e+17\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/epoch 5.99467\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/global_step 3372\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/grad_norm 0.18252\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train/loss 0.5229\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 0.8244\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_runtime 22945.1289\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_samples_per_second 1.177\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: train_steps_per_second 0.147\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33m/content/qwen2-7b\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface/runs/ancw8jgs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/inflaton-ai/huggingface\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240715_143630-ancw8jgs/logs\u001b[0m\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require(\"core\")`! See https://wandb.me/wandb-core for more information.\n" + ] + } + ], + "source": [ + "!llamafactory-cli train train_qwen2_7b.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "background_save": true + }, + "id": "PPHT4JDoIvGk", + "outputId": "868b542e-3cf3-4f96-b3ff-944d48f66e9e" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'/content/drive/MyDrive/runs/qwen2-7b'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import shutil\n", + "shutil.move(\"/content/qwen2-7b\", \"/content/drive/MyDrive/runs\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 35 + }, + "id": "KQolpdAGpUqx", + "outputId": "23443a95-ec97-4633-87d6-18d2f27d979e" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'/content/qwen2-7b'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import shutil\n", + "shutil.move(\"/content/drive/MyDrive/runs\", \"/content/qwen2-7b\", )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mzmNMevzVer3" + }, + "outputs": [], + "source": [ + "def evaluate_model_all_epochs(model_name, adapter_path_base, num_train_epochs, start_epoch=0, load_in_4bit=True, num_of_entries=-1):\n", + " os.environ[\"MODEL_NAME\"] = model_name\n", + " os.environ[\"LOAD_IN_4BIT\"] = \"true\" if load_in_4bit else \"false\"\n", + " for i in range(start_epoch, num_train_epochs + 1):\n", + " print(f\"Epoch {i}\")\n", + " if i == 0:\n", + " os.unsetenv(\"ADAPTER_NAME_OR_PATH\")\n", + " else:\n", + " adapter_path = f\"{adapter_path_base}/checkpoint-{562 * i}\"\n", + " os.environ[\"ADAPTER_NAME_OR_PATH\"] = adapter_path\n", + "\n", + " !python llm_toolkit/eval_logical_reasoning.py {num_of_entries}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3THuVusvVtt8", + "outputId": "2095b621-3aff-4215-f61d-0711acd42e63" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0\n", + "loading env vars from: /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/.env\n", + "Adding /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning to sys.path\n", + "2024-07-16 03:59:05.588323: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2024-07-16 03:59:05.639368: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-07-16 03:59:05.639412: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-07-16 03:59:05.640960: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-07-16 03:59:05.648585: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-16 03:59:06.864846: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "loading /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n", + "Qwen/Qwen2-7B None False datasets/mgtv results/mgtv-results_02_qwen2_7b_colab.csv\n", + "(1) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-7B\n", + "tokenizer_config.json: 100% 1.29k/1.29k [00:00<00:00, 8.65MB/s]\n", + "vocab.json: 100% 2.78M/2.78M [00:00<00:00, 8.33MB/s]\n", + "merges.txt: 100% 1.67M/1.67M [00:00<00:00, 6.20MB/s]\n", + "tokenizer.json: 100% 7.03M/7.03M [00:00<00:00, 15.8MB/s]\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "config.json: 100% 664/664 [00:00<00:00, 4.79MB/s]\n", + "model.safetensors.index.json: 100% 27.8k/27.8k [00:00<00:00, 74.4MB/s]\n", + "Downloading shards: 0% 0/4 [00:00system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|endoftext|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "--------------------------------------------------\n", + "text: 死者受伤了吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 甄庄哭声\n", + "--------------------------------------------------\n", + "puzzle: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "--------------------------------------------------\n", + "truth: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|endoftext|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "Evaluating model: Qwen/Qwen2-7B\n", + " 0% 0/3000 [00:00\n", + " predictions = eval_model(model, tokenizer, datasets[\"test\"])\n", + " File \"/content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\", line 215, in eval_model\n", + " outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n", + " return func(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 1758, in generate\n", + " result = self._sample(\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 2397, in _sample\n", + " outputs = self(\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n", + " return self._call_impl(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n", + " return forward_call(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2/modeling_qwen2.py\", line 1163, in forward\n", + " logits = logits.float()\n", + "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.00 GiB. GPU \n", + "Epoch 1\n", + "loading env vars from: /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/.env\n", + "Adding /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning to sys.path\n", + "2024-07-16 04:36:42.030763: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2024-07-16 04:36:42.082994: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-07-16 04:36:42.083052: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-07-16 04:36:42.084468: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-07-16 04:36:42.092383: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-16 04:36:43.353969: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "loading /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n", + "Qwen/Qwen2-7B /content/qwen2-7b/qwen2-7b/checkpoint-562 False datasets/mgtv results/mgtv-results_02_qwen2_7b_colab.csv\n", + "(1) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-7B\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/vocab.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/merges.txt\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 04:36:49,648 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer_config.json\n", + "[WARNING|logging.py:314] 2024-07-16 04:36:49,914 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/16/2024 04:36:49 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/16/2024 04:36:49 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-16 04:36:50,018 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-16 04:36:50,019 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "07/16/2024 04:36:50 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3474] 2024-07-16 04:36:50,051 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/model.safetensors.index.json\n", + "[INFO|modeling_utils.py:1519] 2024-07-16 04:36:50,054 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:962] 2024-07-16 04:36:50,055 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643\n", + "}\n", + "\n", + "Loading checkpoint shards: 100% 4/4 [00:06<00:00, 1.66s/it]\n", + "[INFO|modeling_utils.py:4280] 2024-07-16 04:36:59,526 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4288] 2024-07-16 04:36:59,526 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-7B.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:917] 2024-07-16 04:36:59,673 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/generation_config.json\n", + "[INFO|configuration_utils.py:962] 2024-07-16 04:36:59,673 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"max_new_tokens\": 2048\n", + "}\n", + "\n", + "07/16/2024 04:37:00 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/16/2024 04:37:01 - INFO - llamafactory.model.adapter - Merged 1 adapter(s).\n", + "07/16/2024 04:37:01 - INFO - llamafactory.model.adapter - Loaded adapter(s): /content/qwen2-7b/qwen2-7b/checkpoint-562\n", + "07/16/2024 04:37:01 - INFO - llamafactory.model.loader - all params: 7,615,616,512\n", + "(2) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "16.521 GB of memory reserved.\n", + "loading train/test data files\n", + "Map: 100% 25000/25000 [00:01<00:00, 22266.31 examples/s]\n", + "Map: 100% 3000/3000 [00:00<00:00, 22229.64 examples/s]\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 25000\n", + " })\n", + " test: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 3000\n", + " })\n", + "})\n", + "--------------------------------------------------\n", + "text: 甄加索是自杀吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 海岸之谜\n", + "--------------------------------------------------\n", + "puzzle: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "--------------------------------------------------\n", + "truth: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上���外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "--------------------------------------------------\n", + "text: 死者受伤了吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: ���庄哭声\n", + "--------------------------------------------------\n", + "puzzle: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "--------------------------------------------------\n", + "truth: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "Evaluating model: Qwen/Qwen2-7B\n", + " 0% 0/3000 [00:00> Setting `pad_token_id` to `eos_token_id`:151643 for open-end generation.\n", + " 0% 0/3000 [30:43\n", + " predictions = eval_model(model, tokenizer, datasets[\"test\"])\n", + " File \"/content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\", line 215, in eval_model\n", + " outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n", + " return func(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 1758, in generate\n", + " result = self._sample(\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 2397, in _sample\n", + " outputs = self(\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n", + " return self._call_impl(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n", + " return forward_call(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2/modeling_qwen2.py\", line 1163, in forward\n", + " logits = logits.float()\n", + "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.92 GiB. GPU \n", + "Epoch 2\n", + "loading env vars from: /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/.env\n", + "Adding /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning to sys.path\n", + "2024-07-16 05:07:51.574401: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2024-07-16 05:07:51.624732: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-07-16 05:07:51.624785: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-07-16 05:07:51.626182: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-07-16 05:07:51.633853: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-16 05:07:52.903770: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "loading /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n", + "Qwen/Qwen2-7B /content/qwen2-7b/qwen2-7b/checkpoint-1124 False datasets/mgtv results/mgtv-results_02_qwen2_7b_colab.csv\n", + "(1) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-7B\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/vocab.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/merges.txt\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:07:59,358 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer_config.json\n", + "[WARNING|logging.py:314] 2024-07-16 05:07:59,635 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/16/2024 05:07:59 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/16/2024 05:07:59 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-16 05:07:59,725 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-16 05:07:59,727 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "07/16/2024 05:07:59 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3474] 2024-07-16 05:07:59,758 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/model.safetensors.index.json\n", + "[INFO|modeling_utils.py:1519] 2024-07-16 05:07:59,761 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:962] 2024-07-16 05:07:59,762 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643\n", + "}\n", + "\n", + "Loading checkpoint shards: 100% 4/4 [00:05<00:00, 1.44s/it]\n", + "[INFO|modeling_utils.py:4280] 2024-07-16 05:08:08,371 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4288] 2024-07-16 05:08:08,371 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-7B.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:917] 2024-07-16 05:08:08,465 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/generation_config.json\n", + "[INFO|configuration_utils.py:962] 2024-07-16 05:08:08,465 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"max_new_tokens\": 2048\n", + "}\n", + "\n", + "07/16/2024 05:08:09 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/16/2024 05:08:09 - INFO - llamafactory.model.adapter - Merged 1 adapter(s).\n", + "07/16/2024 05:08:09 - INFO - llamafactory.model.adapter - Loaded adapter(s): /content/qwen2-7b/qwen2-7b/checkpoint-1124\n", + "07/16/2024 05:08:09 - INFO - llamafactory.model.loader - all params: 7,615,616,512\n", + "(2) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "16.521 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 25000\n", + " })\n", + " test: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 3000\n", + " })\n", + "})\n", + "--------------------------------------------------\n", + "text: 甄加索是自杀吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 海岸之谜\n", + "--------------------------------------------------\n", + "puzzle: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "--------------------------------------------------\n", + "truth: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回���正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "--------------------------------------------------\n", + "text: 死者受伤了吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 甄庄哭声\n", + "--------------------------------------------------\n", + "puzzle: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "--------------------------------------------------\n", + "truth: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "Evaluating model: Qwen/Qwen2-7B\n", + " 0% 0/3000 [00:00> Setting `pad_token_id` to `eos_token_id`:151643 for open-end generation.\n", + " 0% 0/3000 [31:25\n", + " predictions = eval_model(model, tokenizer, datasets[\"test\"])\n", + " File \"/content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\", line 215, in eval_model\n", + " outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n", + " return func(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 1758, in generate\n", + " result = self._sample(\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py\", line 2397, in _sample\n", + " outputs = self(\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n", + " return self._call_impl(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n", + " return forward_call(*args, **kwargs)\n", + " File \"/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2/modeling_qwen2.py\", line 1163, in forward\n", + " logits = logits.float()\n", + "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.92 GiB. GPU \n", + "Epoch 3\n", + "loading env vars from: /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/.env\n", + "Adding /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning to sys.path\n", + "2024-07-16 05:39:41.116319: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2024-07-16 05:39:41.166809: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-07-16 05:39:41.166878: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-07-16 05:39:41.168319: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-07-16 05:39:41.175971: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-16 05:39:42.445909: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "loading /content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\n", + "Qwen/Qwen2-7B /content/qwen2-7b/qwen2-7b/checkpoint-1686 False datasets/mgtv results/mgtv-results_02_qwen2_7b_colab.csv\n", + "(1) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "0.0 GB of memory reserved.\n", + "loading model: Qwen/Qwen2-7B\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,848 >> loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/vocab.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,849 >> loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/merges.txt\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,849 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer.json\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,849 >> loading file added_tokens.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,849 >> loading file special_tokens_map.json from cache at None\n", + "[INFO|tokenization_utils_base.py:2108] 2024-07-16 05:39:48,849 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/tokenizer_config.json\n", + "[WARNING|logging.py:314] 2024-07-16 05:39:49,128 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "07/16/2024 05:39:49 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>\n", + "07/16/2024 05:39:49 - INFO - llamafactory.data.template - Add <|im_start|> to stop words.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n", + "[INFO|configuration_utils.py:733] 2024-07-16 05:39:49,227 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/config.json\n", + "[INFO|configuration_utils.py:796] 2024-07-16 05:39:49,228 >> Model config Qwen2Config {\n", + " \"_name_or_path\": \"Qwen/Qwen2-7B\",\n", + " \"architectures\": [\n", + " \"Qwen2ForCausalLM\"\n", + " ],\n", + " \"attention_dropout\": 0.0,\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"hidden_act\": \"silu\",\n", + " \"hidden_size\": 3584,\n", + " \"initializer_range\": 0.02,\n", + " \"intermediate_size\": 18944,\n", + " \"max_position_embeddings\": 131072,\n", + " \"max_window_layers\": 28,\n", + " \"model_type\": \"qwen2\",\n", + " \"num_attention_heads\": 28,\n", + " \"num_hidden_layers\": 28,\n", + " \"num_key_value_heads\": 4,\n", + " \"rms_norm_eps\": 1e-06,\n", + " \"rope_theta\": 1000000.0,\n", + " \"sliding_window\": 131072,\n", + " \"tie_word_embeddings\": false,\n", + " \"torch_dtype\": \"bfloat16\",\n", + " \"transformers_version\": \"4.41.2\",\n", + " \"use_cache\": true,\n", + " \"use_sliding_window\": false,\n", + " \"vocab_size\": 152064\n", + "}\n", + "\n", + "07/16/2024 05:39:49 - INFO - llamafactory.model.patcher - Using KV cache for faster generation.\n", + "[INFO|modeling_utils.py:3474] 2024-07-16 05:39:49,260 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/model.safetensors.index.json\n", + "[INFO|modeling_utils.py:1519] 2024-07-16 05:39:49,263 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.\n", + "[INFO|configuration_utils.py:962] 2024-07-16 05:39:49,264 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643\n", + "}\n", + "\n", + "Loading checkpoint shards: 100% 4/4 [00:05<00:00, 1.43s/it]\n", + "[INFO|modeling_utils.py:4280] 2024-07-16 05:39:57,929 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.\n", + "\n", + "[INFO|modeling_utils.py:4288] 2024-07-16 05:39:57,929 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-7B.\n", + "If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.\n", + "[INFO|configuration_utils.py:917] 2024-07-16 05:39:58,030 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen2-7B/snapshots/453ed1575b739b5b03ce3758b23befdb0967f40e/generation_config.json\n", + "[INFO|configuration_utils.py:962] 2024-07-16 05:39:58,030 >> Generate config GenerationConfig {\n", + " \"bos_token_id\": 151643,\n", + " \"eos_token_id\": 151643,\n", + " \"max_new_tokens\": 2048\n", + "}\n", + "\n", + "07/16/2024 05:39:58 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.\n", + "07/16/2024 05:39:59 - INFO - llamafactory.model.adapter - Merged 1 adapter(s).\n", + "07/16/2024 05:39:59 - INFO - llamafactory.model.adapter - Loaded adapter(s): /content/qwen2-7b/qwen2-7b/checkpoint-1686\n", + "07/16/2024 05:39:59 - INFO - llamafactory.model.loader - all params: 7,615,616,512\n", + "(2) GPU = NVIDIA L4. Max memory = 22.168 GB.\n", + "16.521 GB of memory reserved.\n", + "loading train/test data files\n", + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 25000\n", + " })\n", + " test: Dataset({\n", + " features: ['text', 'label', 'answer', 'title', 'puzzle', 'truth', 'train_text', 'prompt'],\n", + " num_rows: 3000\n", + " })\n", + "})\n", + "--------------------------------------------------\n", + "text: 甄加索是自杀吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 海岸之谜\n", + "--------------------------------------------------\n", + "puzzle: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "--------------------------------------------------\n", + "truth: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在远离城市喧嚣的海边小屋，一天清晨，邻居发现甄加索僵卧在沙滩上，已无生命迹象。现场没有发现任何打斗的迹象。请问甄加索的死因是什么？\n", + "\n", + "实际情况: 甄加索是一位热爱自然的画家，他每年都会来到这个海边小屋寻找灵感。在他生命的最后几天，他一直在创作一幅描绘海洋生物的画作。在画即将完成的前一天晚上，他骑着自行车外出，打算在海边观赏夜景。然而，他在沙滩上意外发现了一只搁浅的海豚，为了救助这只海豚，他耗费了极大的体力，最终成功将其送回海中。筋疲力尽的甄加索在沙滩上睡着了，由于他患有严重的心脏病，却未告知旁人，在寒冷的海风中，他的心脏停止了跳动。因此，警方在现场只发现了车轮痕迹和未完成的画作，而没有发现任何他杀的迹象。\n", + "\n", + "参与者提出的问题: 甄加索是自杀吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "--------------------------------------------------\n", + "text: 死者受伤了吗\n", + "--------------------------------------------------\n", + "label: 不是\n", + "--------------------------------------------------\n", + "answer: nan\n", + "--------------------------------------------------\n", + "title: 甄庄哭声\n", + "--------------------------------------------------\n", + "puzzle: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "--------------------------------------------------\n", + "truth: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "--------------------------------------------------\n", + "train_text: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "不是<|im_end|>\n", + "--------------------------------------------------\n", + "prompt: <|im_start|>system\n", + "You are an expert in logical reasoning.<|im_end|>\n", + "<|im_start|>user\n", + "你是一个逻辑游戏的主持人。游戏规则如下：\n", + "\n", + "1. 参与者会得到一个谜题。\n", + "2. 参与者可以通过提问来获取线索，尝试解开谜题。\n", + "3. 对于每个问题，主持人将根据实际情况回答以下五个选项之一：是、不是、不重要、回答正确、问法错误。\n", + "4. 回答中不能添加任何其它信息，也不能省略选项中的任何一个字。例如，不可以把“不是”省略成“不”。\n", + "5. 参与者需要根据回答来推理，并最终找出谜题的正确答案。\n", + "\n", + "请严格按照这些规则回答参与者提出的问题。\n", + "\n", + "谜题: 在一个安静的夜晚，小村庄的湖边突然传来了阵阵哭泣声。第二天早晨，村长甄锐发现湖边的石头上放着一顶破旧的帽子，但没有人知道这顶帽子是从哪里来的，哭泣声又是为何。请还原故事真相。\n", + "\n", + "实际情况: 原来，这顶破旧的帽子属于一个小男孩，他小时候与爷爷在湖边生活。爷爷教他钓鱼、游泳，还告诉他湖中的海龟是他们的朋友。后来，小男孩随父母去了城市生活，但每年夏天都会回到村子探望爷爷。然而，去年夏天，爷爷因病去世，小男孩伤心欲绝。今年夏天，他回到村子，来到湖边，想起和爷爷的美好回忆，忍不住哭泣。他将爷爷的帽子放在湖边的石头上，希望能让爷爷的在天之灵得到安慰。那晚的哭泣声正是小男孩在祭莫他亲爱的爷爷。\n", + "\n", + "参与者提出的问题: 死者受伤了吗\n", + "<|im_end|>\n", + "<|im_start|>assistant\n", + "\n", + "Evaluating model: Qwen/Qwen2-7B\n", + " 0% 0/3000 [00:00> Setting `pad_token_id` to `eos_token_id`:151643 for open-end generation.\n", + " 0% 0/3000 [01:33\n", + " predictions = eval_model(model, tokenizer, datasets[\"test\"])\n", + " File \"/content/drive/.shortcut-targets-by-id/1E09lTnfbsjtTgQg65dQ3y9D2R6l8waxR/logical-reasoning/llm_toolkit/logical_reasoning_utils.py\", line 215, in eval_model\n", + " outputs = model.generate(**inputs, max_new_tokens=4096, use_cache=False)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 114, in decorate_context\n", + " with ctx_factory():\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/grad_mode.py\", line 84, in __exit__\n", + " torch.set_grad_enabled(self.prev)\n", + " File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/grad_mode.py\", line 183, in __init__\n", + " def __init__(self, mode: bool) -> None:\n", + "KeyboardInterrupt\n", + "Epoch 4\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "evaluate_model_all_epochs(\"Qwen/Qwen2-7B\", \"/content/qwen2-7b/qwen2-7b\", 4, start_epoch=0, load_in_4bit=False, num_of_entries=-1)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "L4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}