FP8 support??

app.py

@@ -1,5 +1,5 @@
 import streamlit as st
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, FineGrainedFP8Config
 import torch
 import base64
 
@@ -43,6 +43,8 @@ def load_model():
     # model_id = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
     # model_id = "deepseek-ai/deepseek-llm-7b-chat"
     model_id = "deepseek-ai/DeepSeek-V3-0324"
+
+    quantization_config = FineGrainedFP8Config()
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     model = AutoModelForCausalLM.from_pretrained(
         model_id,
@@ -50,7 +52,8 @@ def load_model():
         # torch_dtype=torch.float32
         device_map="auto",
         torch_dtype=torch.float16,
-        trust_remote_code = True
+        trust_remote_code = True,
+        quantization_config=quantization_config
     )
     # model.to("cpu")
     return tokenizer, model