view if eos_token_id is set
Browse files
app.py
CHANGED
@@ -31,6 +31,10 @@ terminators = [
|
|
31 |
llama_tokenizer.eos_token_id,
|
32 |
llama_tokenizer.convert_tokens_to_ids("<|eot_id|>")
|
33 |
]
|
|
|
|
|
|
|
|
|
34 |
|
35 |
# Place just input pass and return generation output
|
36 |
def llama_generation(input_text: str,
|
|
|
31 |
llama_tokenizer.eos_token_id,
|
32 |
llama_tokenizer.convert_tokens_to_ids("<|eot_id|>")
|
33 |
]
|
34 |
+
# Check if eos_token_id is valid
|
35 |
+
if llama_tokenizer.eos_token_id is None or llama_tokenizer.eos_token_id == llama_tokenizer.pad_token_id:
|
36 |
+
raise ValueError("Invalid eos_token_id. Please check your tokenizer configuration.")
|
37 |
+
|
38 |
|
39 |
# Place just input pass and return generation output
|
40 |
def llama_generation(input_text: str,
|