Upload folder using huggingface_hub

README.md

@@ -13,14 +13,24 @@ This tiny model is for debugging. It is randomly initialized with the config ada
 ### Example usage:
 
 ```python
+import unittest
+
+import torch
+
 import soundfile as sf
 from qwen_omni_utils import process_mm_info
-from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor
+from transformers import (
+    Qwen2_5OmniForConditionalGeneration,
+    Qwen2_5OmniPreTrainedModel,
+    Qwen2_5OmniProcessor,
+)
 
 model_id = "tiny-random/qwen2.5-omni"
 # model = Qwen2_5OmniModel.from_pretrained(model_id, torch_dtype="auto", device_map="auto").eval()
 # We recommend enabling flash_attention_2 for better acceleration and memory saving.
-model = Qwen2_5OmniModel.from_pretrained(
+
+Qwen2_5OmniPreTrainedModel._init_weights = unittest.mock.Mock()
+model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
     model_id,
     torch_dtype="auto",
     device_map="auto",
@@ -31,14 +41,16 @@ processor = Qwen2_5OmniProcessor.from_pretrained(model_id)
 conversation = [
     {
         "role": "system",
-        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+        "content": [
+            {"type": "text", "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."}
+        ],
     },
     {
         "role": "user",
         "content": [
             {"type": "text", "text": "Hi, can you tell me a joke?"},
-            {"type": "audio", "audio": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Audio/glass-breaking-151256.mp3"},
-            {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
+            # {"type": "audio", "audio": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Audio/glass-breaking-151256.mp3"},
+            # {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
             {"type": "image", "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"},
         ],
     },
@@ -57,6 +69,7 @@ inputs = inputs.to(model.device).to(model.dtype)
 text_ids, audio = model.generate(
     **inputs, use_audio_in_video=True,
     thinker_max_new_tokens=16, talker_max_new_tokens=16,
+    temperature=0.1,
 )
 
 text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
@@ -71,17 +84,20 @@ sf.write(
 ### Codes to create this repo:
 
 ```python
+import unittest
 from pathlib import Path
 
 import torch
 
+import accelerate
 from huggingface_hub import hf_hub_download
 from transformers import (
     AutoConfig,
     AutoModelForCausalLM,
     AutoTokenizer,
     GenerationConfig,
-    Qwen2_5OmniModel,
+    Qwen2_5OmniForConditionalGeneration,
+    Qwen2_5OmniPreTrainedModel,
     Qwen2_5OmniProcessor,
     pipeline,
     set_seed,
@@ -166,8 +182,11 @@ for _, info in spk_dict.items():
     info['cond'] = info['cond'][:, :config.token2wav_config.dit_config.enc_emb_dim].clone()
 torch.save(spk_dict, Path(save_folder, "spk_dict.pt"))
 
+# patch for non-affine layernorm
+Qwen2_5OmniPreTrainedModel._init_weights = unittest.mock.Mock()
+
 torch.set_default_dtype(torch.bfloat16)
-model = Qwen2_5OmniModel(
+model = Qwen2_5OmniForConditionalGeneration(
     config,
 )
 torch.set_default_dtype(torch.float32)

chat_template.jinja

@@ -0,0 +1,7 @@
+{% set audio_count = namespace(value=0) %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_bos|><|IMAGE|><|vision_eos|>{% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}{% set audio_count.value = audio_count.value + 1 %}{% if add_audio_id %}Audio {{ audio_count.value }}: {% endif %}<|audio_bos|><|AUDIO|><|audio_eos|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_bos|><|VIDEO|><|vision_eos|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

config.json

@@ -1,13 +1,12 @@
 {
   "architectures": [
-    "Qwen2_5OmniModel"
+    "Qwen2_5OmniForConditionalGeneration"
   ],
   "enable_audio_output": true,
   "enable_talker": true,
   "model_type": "qwen2_5_omni",
   "talker_config": {
     "_attn_implementation_autoset": true,
-    "_name_or_path": "Qwen2.5-Omni-7B/talker",
     "architectures": [
       "Qwen2OmniTalkerForConditionalGeneration"
     ],
@@ -61,13 +60,11 @@
   },
   "thinker_config": {
     "_attn_implementation_autoset": true,
-    "_name_or_path": "Qwen2.5-Omni-7B/thinker",
     "architectures": [
       "Qwen2OmniNaViTThinkerForConditionalGeneration"
     ],
     "audio_config": {
       "_attn_implementation_autoset": true,
-      "_name_or_path": "",
       "activation_dropout": 0.0,
       "activation_function": "gelu",
       "add_cross_attention": false,
@@ -99,6 +96,7 @@
         "1": "LABEL_1"
       },
       "init_std": 0.02,
+      "initializer_range": 0.02,
       "is_decoder": false,
       "is_encoder_decoder": false,
       "label2id": {
@@ -153,13 +151,13 @@
     "ignore_index": -100,
     "image_token_index": 151655,
     "init_std": 0.02,
+    "initializer_range": 0.02,
     "model_type": "qwen2_5_omni_thinker",
     "pad_token_id": 151643,
     "position_id_per_seconds": 25,
     "seconds_per_chunk": 2,
     "text_config": {
       "_attn_implementation_autoset": false,
-      "_name_or_path": "",
       "add_cross_attention": false,
       "architectures": null,
       "attention_dropout": 0.0,
@@ -185,6 +183,7 @@
         "1": "LABEL_1"
       },
       "init_std": 0.02,
+      "initializer_range": 0.02,
       "intermediate_size": 32,
       "is_decoder": false,
       "is_encoder_decoder": false,
@@ -251,7 +250,6 @@
     "video_token_index": 151656,
     "vision_config": {
       "_attn_implementation_autoset": true,
-      "_name_or_path": "",
       "add_cross_attention": false,
       "architectures": null,
       "bad_words_ids": null,
@@ -283,6 +281,7 @@
       "in_channels": 3,
       "in_chans": 3,
       "init_std": 0.02,
+      "initializer_range": 0.02,
       "intermediate_size": 32,
       "is_decoder": false,
       "is_encoder_decoder": false,
@@ -340,7 +339,6 @@
     "_attn_implementation_autoset": true,
     "bigvgan_config": {
       "_attn_implementation_autoset": true,
-      "_name_or_path": "",
       "add_cross_attention": false,
       "architectures": null,
       "bad_words_ids": null,
@@ -425,7 +423,6 @@
     },
     "dit_config": {
       "_attn_implementation_autoset": true,
-      "_name_or_path": "",
       "add_cross_attention": false,
       "architectures": null,
       "bad_words_ids": null,
@@ -534,5 +531,5 @@
     "model_type": "qwen2_5_omni_token2wav"
   },
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.50.0.dev0"
+  "transformers_version": "4.52.0.dev0"
 }

generation_config.json

@@ -1,4 +1,4 @@
 {
   "_from_model_config": true,
-  "transformers_version": "4.50.0.dev0"
+  "transformers_version": "4.52.0.dev0"
 }

tokenizer_config.json

@@ -197,7 +197,6 @@
   "audio_eos_token": "<|audio_eos|>",
   "audio_token": "<|AUDIO|>",
   "bos_token": null,
-  "chat_template": "{% set audio_count = namespace(value=0) %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_bos|><|IMAGE|><|vision_eos|>{% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}{% set audio_count.value = audio_count.value + 1 %}{% if add_audio_id %}Audio {{ audio_count.value }}: {% endif %}<|audio_bos|><|AUDIO|><|audio_eos|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_bos|><|VIDEO|><|vision_eos|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",