| # import tiktoken | |
| # from typing import Union | |
| # def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]: | |
| # encoding = tiktoken.encoding_for_model(model) | |
| # encoded = encoding.encode(text) | |
| # num_tokens = len(encoded) | |
| # return num_tokens, encoded |