update
Browse files
app.py
CHANGED
|
@@ -37,9 +37,6 @@ terminators = [
|
|
| 37 |
special_tokens = llama_tokenizer.special_tokens_map
|
| 38 |
eos_token = special_tokens.get("eos_token")
|
| 39 |
|
| 40 |
-
# Get the default EOS token from the tokenizer
|
| 41 |
-
eos_token = llama_tokenizer.eos_token
|
| 42 |
-
|
| 43 |
print("Default EOS Token:", eos_token)
|
| 44 |
|
| 45 |
|
|
|
|
| 37 |
special_tokens = llama_tokenizer.special_tokens_map
|
| 38 |
eos_token = special_tokens.get("eos_token")
|
| 39 |
|
|
|
|
|
|
|
|
|
|
| 40 |
print("Default EOS Token:", eos_token)
|
| 41 |
|
| 42 |
|