tokenizer/apply_tokenizer.py

from tokenizers import ByteLevelBPETokenizer

tokenizer = ByteLevelBPETokenizer.from_file(
    "./salesforce/codet5-vocab.json",
    "./salesforce/codet5-merges.txt"
)
tokenizer.add_special_tokens([
    "<pad>",
    "<s>",
    "</s>",
    "<unk>",
    "<mask>"
])

print(
    tokenizer.encode("<s> hello <unk> Don't you love 🤗 Transformers <mask> yes . </s>").tokens
)