fd3769e
1
2
3
4
5
6
7
8
9
10
from transformers import PreTrainedTokenizerBase class BaseSequenceTokenizer: def __init__(self, tokenizer: PreTrainedTokenizerBase): self.tokenizer = tokenizer def __call__(self, sequences, **kwargs): raise NotImplementedError
