stefan-it commited on
Commit
0739132
·
verified ·
1 Parent(s): eb87a21

feat: add tokenizer config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +20 -0
tokenizer_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|bos|>",
4
+ "eos_token": "<|assistant_end|>",
5
+ "pad_token": "<|assistant_end|>",
6
+ "additional_special_tokens": [
7
+ "<|user_start|>",
8
+ "<|user_end|>",
9
+ "<|assistant_start|>",
10
+ "<|python_start|>",
11
+ "<|python_end|>",
12
+ "<|output_start|>",
13
+ "<|output_end|>"
14
+ ],
15
+ "chat_template": "chat_template.jinja",
16
+ "model_input_names": [
17
+ "input_ids",
18
+ "attention_mask"
19
+ ]
20
+ }