Files
engine/scripts/get_tokenizers_json.py

6 lines
142 B
Python

from transformers import AutoTokenizer
model = input("Enter model ID: ")
tok = AutoTokenizer.from_pretrained(model)
tok.save_pretrained(".")