update

README.md

@@ -50,7 +50,7 @@ This model is not owned or developed by NVIDIA. This model has been developed an
 * Linux <br>
 
 ## Model Version(s):
-The model is quantized with nvidia-modelopt **v0.15.1**  <br>
+The model is quantized with nvidia-modelopt **v0.27.0**  <br>
 
 ## Datasets:
 * Calibration Dataset: [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) <br>
@@ -164,4 +164,4 @@ for output in outputs:
 
 ```
 
-This model can be deployed with an OpenAI Compatible Server via the vLLM backend. Instructions [here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).
+This model can be deployed with an OpenAI Compatible Server via the vLLM backend. Instructions [here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": "/opt/zhiyu/ckpts/Meta-Llama-3.1-8B-Instruct/",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -11,6 +10,7 @@
     128008,
     128009
   ],
+  "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
@@ -33,7 +33,7 @@
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.44.0",
+  "transformers_version": "4.50.0.dev0",
   "use_cache": true,
   "vocab_size": 128256
 }

generation_config.json

@@ -8,5 +8,5 @@
   ],
   "temperature": 0.6,
   "top_p": 0.9,
-  "transformers_version": "4.44.0"
+  "transformers_version": "4.50.0.dev0"
 }

hf_quant_config.json

@@ -1,7 +1,7 @@
 {
     "producer": {
         "name": "modelopt",
-        "version": "0.23.0"
+        "version": "0.27.0"
     },
     "quantization": {
         "quant_algo": "FP8",

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d44feba7a89792817523bcd648e695cfd457c2ce41b67dcdea4a01b9a59d1f3
-size 4997857356
+oid sha256:499ce93537438acbb81ac5bf6f6890d0501b572d85405866705b700f08644a24
+size 4997857256

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ade51d4b47f5bab5e04c110514351ca8a1fa176055fb1bdab4e4844bf18e8b8
-size 4083429684
+oid sha256:62e7244fe8487c09413cd9ed683d94d580dca68a4562549f7b62c8f8c0ba5246
+size 4083429784

model.safetensors.index.json

@@ -267,7 +267,7 @@
     "model.layers.18.mlp.up_proj.weight_scale": "model-00002-of-00002.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.18.self_attn.k_proj.input_scale": "model-00001-of-00002.safetensors",
-    "model.layers.18.self_attn.k_proj.k_scale": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.k_scale": "model-00002-of-00002.safetensors",
     "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.k_proj.weight_scale": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.o_proj.input_scale": "model-00002-of-00002.safetensors",

tokenizer_config.json

@@ -2053,11 +2053,12 @@
   "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 131072,
   "pad_token": "<|eot_id|>",
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "tokenizer_class": "PreTrainedTokenizer"
 }