Raiff1982 commited on
Commit
ed16de9
·
1 Parent(s): 17cb643
Files changed (1) hide show
  1. AICoreAGIX_with_TB.py +11 -12
AICoreAGIX_with_TB.py CHANGED
@@ -100,21 +100,20 @@ class AICoreAGIX:
100
  )
101
  except (ValueError, KeyError) as e:
102
  logger.warning(f"[Tokenizer Load]: Remote code failed — falling back. Reason: {e}")
103
- try:
104
- return AutoTokenizer.from_pretrained(
105
- model_name,
106
- trust_remote_code=False,
107
- use_fast=False
108
- )
109
- except Exception as e2:
110
- logger.warning(f"[Tokenizer Load]: Full fallback to gpt2. Reason: {e2}")
111
- return AutoTokenizer.from_pretrained("gpt2")
112
  try:
113
- return AutoTokenizer.from_pretrained("gpt2", local_files_only=True)
114
- except Exception as e3:
 
 
 
 
 
 
 
 
115
  logger.error(f"[Tokenizer Load]: Even fallback to GPT2 failed: {e3}")
116
  raise RuntimeError("Tokenizer load completely failed.")
117
-
118
  def _safe_load_model(self, model_name):
119
  try:
120
  return AutoModelForCausalLM.from_pretrained(
 
100
  )
101
  except (ValueError, KeyError) as e:
102
  logger.warning(f"[Tokenizer Load]: Remote code failed — falling back. Reason: {e}")
 
 
 
 
 
 
 
 
 
103
  try:
104
+ return AutoTokenizer.from_pretrained(
105
+ model_name,
106
+ trust_remote_code=False,
107
+ use_fast=False
108
+ )
109
+ except Exception as e2:
110
+ logger.warning(f"[Tokenizer Load]: Full fallback to gpt2 failed: {e2}")
111
+ try:
112
+ return AutoTokenizer.from_pretrained("gpt2", local_files_only=True)
113
+ except Exception as e3:
114
  logger.error(f"[Tokenizer Load]: Even fallback to GPT2 failed: {e3}")
115
  raise RuntimeError("Tokenizer load completely failed.")
116
+
117
  def _safe_load_model(self, model_name):
118
  try:
119
  return AutoModelForCausalLM.from_pretrained(