Builds custom tokenizer using subset of epidemiological data

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.processors import ByteLevel as ByteLevelProcessor

tokenizer = Tokenizer(BPE())

tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)

trainer = BpeTrainer(
    vocab_size=32768,
    special_tokens=[
        "<|end_of_text|>"
    ],
    show_progress=True
)

corpus_files = ["mmwr_sample.txt"]

tokenizer.train(corpus_files, trainer)