colin
/
gpt-pretrain
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use local tokenizer.

This commit is contained in:
Colin 2024-02-24 14:14:12 +08:00
parent ac61c4d925
commit 122cbd9ff8
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
custom_models/gpt2/tokenizer_config.json Normal file
View File

@ -0,0 +1 @@
{"model_max_length": 1024}

2
lit_train.py
View File

@ -182,7 +182,7 @@ if __name__ == "__main__":
lit_module = LitModule(args.model_name, "./custom_models/gpt2", args.learning_rate, args.use_tril_attention_mask)
# datasets
tokenizer = load_tokenizer(args.tokenizer_name_or_path)
tokenizer = load_tokenizer("./custom_models/gpt2")
train_dataset_list = []
val_dataset_list = []
for dataset_name in args.dataset_name: