update
Browse files
app.py
CHANGED
|
@@ -37,7 +37,7 @@ special_tokens = llama_tokenizer.special_tokens_map
|
|
| 37 |
eos_token = special_tokens.get("eos_token")
|
| 38 |
|
| 39 |
# Get the default EOS token from the tokenizer
|
| 40 |
-
eos_token =
|
| 41 |
|
| 42 |
print("Default EOS Token:", eos_token)
|
| 43 |
|
|
|
|
| 37 |
eos_token = special_tokens.get("eos_token")
|
| 38 |
|
| 39 |
# Get the default EOS token from the tokenizer
|
| 40 |
+
eos_token = llama_tokenizer.eos_token
|
| 41 |
|
| 42 |
print("Default EOS Token:", eos_token)
|
| 43 |
|