fix: fix merges in tokenizer
Browse files
moonshine_tiny_tokenizer.json
CHANGED
|
@@ -15703,7 +15703,7 @@
|
|
| 15703 |
"\"?": 8652,
|
| 15704 |
"▁>>>": 8653,
|
| 15705 |
"Que": 8654,
|
| 15706 |
-
"
|
| 15707 |
"▁plain": 8656,
|
| 15708 |
"ativa": 8657,
|
| 15709 |
"ocker": 8658,
|
|
@@ -24939,7 +24939,7 @@
|
|
| 24939 |
"▁farm": 17888,
|
| 24940 |
"▁rôle": 17889,
|
| 24941 |
"▁статьи": 17890,
|
| 24942 |
-
"
|
| 24943 |
"subfigure": 17892,
|
| 24944 |
"èces": 17893,
|
| 24945 |
"ziel": 17894,
|
|
@@ -27294,7 +27294,7 @@
|
|
| 27294 |
"▁gcc": 20243,
|
| 27295 |
"▁scène": 20244,
|
| 27296 |
"Navigation": 20245,
|
| 27297 |
-
"▁
|
| 27298 |
"▁кан": 20247,
|
| 27299 |
"▁towns": 20248,
|
| 27300 |
"Username": 20249,
|
|
@@ -37129,7 +37129,7 @@
|
|
| 37129 |
"æ": 30078,
|
| 37130 |
"њ": 30079,
|
| 37131 |
" ": 30080,
|
| 37132 |
-
"
|
| 37133 |
"Э": 30082,
|
| 37134 |
"ë": 30083,
|
| 37135 |
"õ": 30084,
|
|
@@ -57967,7 +57967,7 @@
|
|
| 57967 |
"▁>> >",
|
| 57968 |
"Qu e",
|
| 57969 |
"Q ue",
|
| 57970 |
-
"
|
| 57971 |
"▁p lain",
|
| 57972 |
"▁pl ain",
|
| 57973 |
"▁pla in",
|
|
@@ -77192,7 +77192,7 @@
|
|
| 77192 |
"▁fa rm",
|
| 77193 |
"▁r ôle",
|
| 77194 |
"▁стать и",
|
| 77195 |
-
"
|
| 77196 |
"sub figure",
|
| 77197 |
"èce s",
|
| 77198 |
"è ces",
|
|
@@ -81855,7 +81855,7 @@
|
|
| 81855 |
"▁ gcc",
|
| 81856 |
"▁sc ène",
|
| 81857 |
"N avigation",
|
| 81858 |
-
"▁
|
| 81859 |
"▁к ан",
|
| 81860 |
"▁ка н",
|
| 81861 |
"▁ кан",
|
|
@@ -100301,4 +100301,4 @@
|
|
| 100301 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
| 100302 |
]
|
| 100303 |
}
|
| 100304 |
-
}
|
|
|
|
| 15703 |
"\"?": 8652,
|
| 15704 |
"▁>>>": 8653,
|
| 15705 |
"Que": 8654,
|
| 15706 |
+
" ": 8655,
|
| 15707 |
"▁plain": 8656,
|
| 15708 |
"ativa": 8657,
|
| 15709 |
"ocker": 8658,
|
|
|
|
| 24939 |
"▁farm": 17888,
|
| 24940 |
"▁rôle": 17889,
|
| 24941 |
"▁статьи": 17890,
|
| 24942 |
+
" ": 17891,
|
| 24943 |
"subfigure": 17892,
|
| 24944 |
"èces": 17893,
|
| 24945 |
"ziel": 17894,
|
|
|
|
| 27294 |
"▁gcc": 20243,
|
| 27295 |
"▁scène": 20244,
|
| 27296 |
"Navigation": 20245,
|
| 27297 |
+
"▁ ": 20246,
|
| 27298 |
"▁кан": 20247,
|
| 27299 |
"▁towns": 20248,
|
| 27300 |
"Username": 20249,
|
|
|
|
| 37129 |
"æ": 30078,
|
| 37130 |
"њ": 30079,
|
| 37131 |
" ": 30080,
|
| 37132 |
+
" ": 30081,
|
| 37133 |
"Э": 30082,
|
| 37134 |
"ë": 30083,
|
| 37135 |
"õ": 30084,
|
|
|
|
| 57967 |
"▁>> >",
|
| 57968 |
"Qu e",
|
| 57969 |
"Q ue",
|
| 57970 |
+
" ",
|
| 57971 |
"▁p lain",
|
| 57972 |
"▁pl ain",
|
| 57973 |
"▁pla in",
|
|
|
|
| 77192 |
"▁fa rm",
|
| 77193 |
"▁r ôle",
|
| 77194 |
"▁стать и",
|
| 77195 |
+
" ",
|
| 77196 |
"sub figure",
|
| 77197 |
"èce s",
|
| 77198 |
"è ces",
|
|
|
|
| 81855 |
"▁ gcc",
|
| 81856 |
"▁sc ène",
|
| 81857 |
"N avigation",
|
| 81858 |
+
"▁ ",
|
| 81859 |
"▁к ан",
|
| 81860 |
"▁ка н",
|
| 81861 |
"▁ кан",
|
|
|
|
| 100301 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
| 100302 |
]
|
| 100303 |
}
|
| 100304 |
+
}
|