inferencerlabs commited on
Commit
557ca23
·
verified ·
1 Parent(s): 2db4040

Upload model file

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +64 -0
tokenizer_config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "]~!b[",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "[e~[",
7
+ "extra_special_tokens": [
8
+ "<code_interpreter>",
9
+ "<commit_after>",
10
+ "<commit_before>",
11
+ "<commit_msg>",
12
+ "<empty_output>",
13
+ "<filename>",
14
+ "<fim_middle>",
15
+ "<fim_pad>",
16
+ "<fim_prefix>",
17
+ "<fim_suffix>",
18
+ "<function_call>",
19
+ "<gh_stars>",
20
+ "]<]speech[>[",
21
+ "]<]image[>[",
22
+ "]<]video[>[",
23
+ "]<]start of speech[>[",
24
+ "]<]end of speech[>[",
25
+ "]<]start of image[>[",
26
+ "]<]end of image[>[",
27
+ "]<]start of video[>[",
28
+ "]<]end of video[>[",
29
+ "]<]vision pad[>[",
30
+ "]~!b[",
31
+ "<issue_closed>",
32
+ "<issue_comment>",
33
+ "<issue_start>",
34
+ "<jupyter_code>",
35
+ "<jupyter_output>",
36
+ "<jupyter_start>",
37
+ "<jupyter_text>",
38
+ "<reponame>",
39
+ "[e~[",
40
+ "]!d~[",
41
+ "]!p~[",
42
+ "]~b]",
43
+ "<jupyter_error>",
44
+ "<add_file>",
45
+ "<delete_file>",
46
+ "<rename_file>",
47
+ "<edit_file>",
48
+ "<commit_message>",
49
+ "<empty_source_file>",
50
+ "<repo_struct>",
51
+ "<code_context>",
52
+ "<file_content>",
53
+ "<source_files>",
54
+ "<pr_start>",
55
+ "<review_comment>",
56
+ "<filepath>",
57
+ "<file_sep>"
58
+ ],
59
+ "is_local": true,
60
+ "model_max_length": 40960000,
61
+ "tokenizer_class": "TokenizersBackend",
62
+ "tool_parser_type": "minimax_m2",
63
+ "unk_token": "]!d~["
64
+ }