sandz7 commited on
Commit
3e7a6d0
Β·
1 Parent(s): e8bc467

view if eos_token_id is set

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -31,6 +31,10 @@ terminators = [
31
  llama_tokenizer.eos_token_id,
32
  llama_tokenizer.convert_tokens_to_ids("<|eot_id|>")
33
  ]
 
 
 
 
34
 
35
  # Place just input pass and return generation output
36
  def llama_generation(input_text: str,
 
31
  llama_tokenizer.eos_token_id,
32
  llama_tokenizer.convert_tokens_to_ids("<|eot_id|>")
33
  ]
34
+ # Check if eos_token_id is valid
35
+ if llama_tokenizer.eos_token_id is None or llama_tokenizer.eos_token_id == llama_tokenizer.pad_token_id:
36
+ raise ValueError("Invalid eos_token_id. Please check your tokenizer configuration.")
37
+
38
 
39
  # Place just input pass and return generation output
40
  def llama_generation(input_text: str,