pt-tupi-tokenizer / tokenizer.json

Upload tokenizer

c59e1e6 verified 4 months ago

18.7 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": true,
	"special": true
	},
	{
	"id": 1,
	"content": "<pad>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": true,
	"special": true
	},
	{
	"id": 2,
	"content": "</s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": true,
	"special": true
	},
	{
	"id": 3,
	"content": "<unk>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": true,
	"special": true
	},
	{
	"id": 4,
	"content": "<mask>",
	"single_word": false,
	"lstrip": true,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": true
	},
	"post_processor": {
	"type": "RobertaProcessing",
	"sep": [
	"</s>",
	2
	],
	"cls": [
	"<s>",
	0
	],
	"trim_offsets": true,
	"add_prefix_space": false
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": "",
	"end_of_word_suffix": "",
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<s>": 0,
	"<pad>": 1,
	"</s>": 2,
	"<unk>": 3,
	"<mask>": 4,
	"!": 5,
	"\"": 6,
	"#": 7,
	"$": 8,
	"%": 9,
	"&": 10,
	"'": 11,
	"(": 12,
	")": 13,
	"*": 14,
	"+": 15,
	",": 16,
	"-": 17,
	".": 18,
	"/": 19,
	"0": 20,
	"1": 21,
	"2": 22,
	"3": 23,
	"4": 24,
	"5": 25,
	"6": 26,
	"7": 27,
	"8": 28,
	"9": 29,
	":": 30,
	";": 31,
	"<": 32,
	"=": 33,
	">": 34,
	"?": 35,
	"@": 36,
	"A": 37,
	"B": 38,
	"C": 39,
	"D": 40,
	"E": 41,
	"F": 42,
	"G": 43,
	"H": 44,
	"I": 45,
	"J": 46,
	"K": 47,
	"L": 48,
	"M": 49,
	"N": 50,
	"O": 51,
	"P": 52,
	"Q": 53,
	"R": 54,
	"S": 55,
	"T": 56,
	"U": 57,
	"V": 58,
	"W": 59,
	"X": 60,
	"Y": 61,
	"Z": 62,
	"[": 63,
	"\\": 64,
	"]": 65,
	"^": 66,
	"_": 67,
	"`": 68,
	"a": 69,
	"b": 70,
	"c": 71,
	"d": 72,
	"e": 73,
	"f": 74,
	"g": 75,
	"h": 76,
	"i": 77,
	"j": 78,
	"k": 79,
	"l": 80,
	"m": 81,
	"n": 82,
	"o": 83,
	"p": 84,
	"q": 85,
	"r": 86,
	"s": 87,
	"t": 88,
	"u": 89,
	"v": 90,
	"w": 91,
	"x": 92,
	"y": 93,
	"z": 94,
	"{": 95,
	"\|": 96,
	"}": 97,
	"~": 98,
	"¡": 99,
	"¢": 100,
	"£": 101,
	"¤": 102,
	"¥": 103,
	"¦": 104,
	"§": 105,
	"¨": 106,
	"©": 107,
	"ª": 108,
	"«": 109,
	"¬": 110,
	"®": 111,
	"¯": 112,
	"°": 113,
	"±": 114,
	"²": 115,
	"³": 116,
	"´": 117,
	"µ": 118,
	"¶": 119,
	"·": 120,
	"¸": 121,
	"¹": 122,
	"º": 123,
	"»": 124,
	"¼": 125,
	"½": 126,
	"¾": 127,
	"¿": 128,
	"À": 129,
	"Á": 130,
	"Â": 131,
	"Ã": 132,
	"Ä": 133,
	"Å": 134,
	"Æ": 135,
	"Ç": 136,
	"È": 137,
	"É": 138,
	"Ê": 139,
	"Ë": 140,
	"Ì": 141,
	"Í": 142,
	"Î": 143,
	"Ï": 144,
	"Ð": 145,
	"Ñ": 146,
	"Ò": 147,
	"Ó": 148,
	"Ô": 149,
	"Õ": 150,
	"Ö": 151,
	"×": 152,
	"Ø": 153,
	"Ù": 154,
	"Ú": 155,
	"Û": 156,
	"Ü": 157,
	"Ý": 158,
	"Þ": 159,
	"ß": 160,
	"à": 161,
	"á": 162,
	"â": 163,
	"ã": 164,
	"ä": 165,
	"å": 166,
	"æ": 167,
	"ç": 168,
	"è": 169,
	"é": 170,
	"ê": 171,
	"ë": 172,
	"ì": 173,
	"í": 174,
	"î": 175,
	"ï": 176,
	"ð": 177,
	"ñ": 178,
	"ò": 179,
	"ó": 180,
	"ô": 181,
	"õ": 182,
	"ö": 183,
	"÷": 184,
	"ø": 185,
	"ù": 186,
	"ú": 187,
	"û": 188,
	"ü": 189,
	"ý": 190,
	"þ": 191,
	"ÿ": 192,
	"Ā": 193,
	"ā": 194,
	"Ă": 195,
	"ă": 196,
	"Ą": 197,
	"ą": 198,
	"Ć": 199,
	"ć": 200,
	"Ĉ": 201,
	"ĉ": 202,
	"Ċ": 203,
	"ċ": 204,
	"Č": 205,
	"č": 206,
	"Ď": 207,
	"ď": 208,
	"Đ": 209,
	"đ": 210,
	"Ē": 211,
	"ē": 212,
	"Ĕ": 213,
	"ĕ": 214,
	"Ė": 215,
	"ė": 216,
	"Ę": 217,
	"ę": 218,
	"Ě": 219,
	"ě": 220,
	"Ĝ": 221,
	"ĝ": 222,
	"Ğ": 223,
	"ğ": 224,
	"Ġ": 225,
	"ġ": 226,
	"Ģ": 227,
	"ģ": 228,
	"Ĥ": 229,
	"ĥ": 230,
	"Ħ": 231,
	"ħ": 232,
	"Ĩ": 233,
	"ĩ": 234,
	"Ī": 235,
	"ī": 236,
	"Ĭ": 237,
	"ĭ": 238,
	"Į": 239,
	"į": 240,
	"İ": 241,
	"ı": 242,
	"Ĳ": 243,
	"ĳ": 244,
	"Ĵ": 245,
	"ĵ": 246,
	"Ķ": 247,
	"ķ": 248,
	"ĸ": 249,
	"Ĺ": 250,
	"ĺ": 251,
	"Ļ": 252,
	"ļ": 253,
	"Ľ": 254,
	"ľ": 255,
	"Ŀ": 256,
	"ŀ": 257,
	"Ł": 258,
	"ł": 259,
	"Ń": 260,
	"ra": 261,
	"an": 262,
	"Ã¡": 263,
	"ba": 264,
	"ca": 265,
	"ua": 266,
	"Ã©": 267,
	"Ã§": 268,
	"ru": 269,
	"ana": 270,
	"Ca": 271,
	"im": 272,
	"Ara": 273,
	"na": 274,
	"ri": 275,
	"ti": 276,
	"ap": 277,
	"Ãª": 278,
	"ÃŃ": 279,
	"Am": 280,
	"ara": 281,
	"Ã£": 282,
	"Aba": 283,
	"Ġ(": 284,
	"Ja": 285,
	"uara": 286,
	"au": 287,
	"rÃ©": 288,
	"uru": 289,
	"mb": 290,
	"ng": 291,
	"rÃ¡": 292,
	"Ã§a": 293,
	"An": 294,
	"Ti": 295,
	"ira": 296,
	"ma": 297,
	"ta": 298,
	"Ka": 299,
	"bi": 300,
	"bu": 301,
	"cu": 302,
	"ni": 303,
	"ui": 304,
	"va": 305,
	"wa": 306,
	"Ana": 307,
	"bÃ¡": 308,
	"et": 309,
	"he": 310,
	"api": 311,
	"nga": 312,
	"Ai": 313,
	"Aj": 314,
	"Ju": 315,
	"Ma": 316,
	"Man": 317,
	"Para": 318,
	"eba": 319,
	"gua": 320,
	"mim": 321,
	"oca": 322,
	"quara": 323,
	"uÃ£": 324,
	"Ãº": 325,
	"Cari": 326,
	"Amana": 327,
	"Ap": 328,
	"Aca": 329,
	"Aim": 330,
	"Ira": 331,
	"Ita": 332,
	"Juru": 333,
	"Ya": 334,
	"rÃª": 335,
	"ss": 336,
	"ue": 337,
	"uÃ§": 338,
	"xana": 339,
	"and": 340,
	"ang": 341,
	"anÃ£": 342,
	"Arac": 343,
	"auÃ¡": 344,
	"Ã§aÃŃ": 345,
	"apixana": 346,
	"Ab": 347,
	"Ay": 348,
	"Awa": 349,
	"Gua": 350,
	"Guara": 351,
	"Nhe": 352,
	"Oca": 353,
	"Pe": 354,
	"Pi": 355,
	"Tu": 356,
	"Xim": 357,
	"arÃ¡": 358,
	"cÃ¡": 359,
	"di": 360,
	"eru": 361,
	"emb": 362,
	"it": 363,
	"ju": 364,
	"ka": 365,
	"mu": 366,
	"mÃŃ": 367,
	"oti": 368,
	"pi": 369,
	"po": 370,
	"ry": 371,
	"tÃª": 372,
	"uÃ§a": 373,
	"Ã³": 374,
	"camim": 375,
	"Ã§u": 376,
	"ÃŃba": 377,
	"Abaca": 378,
	"biquara": 379,
	"burÃ©": 380,
	"Maca": 381,
	"Mani": 382,
	"guar": 383,
	"Ãºna": 384,
	"Tupi": 385,
	"Av": 386,
	"Ati": 387,
	"Ava": 388,
	"Da": 389,
	"Ia": 390,
	"Iba": 391,
	"Ibi": 392,
	"Jap": 393,
	"Lau": 394,
	"Mu": 395,
	"Mana": 396,
	"PuÃ§": 397,
	"ParÃ¡": 398,
	"Poti": 399,
	"Qui": 400,
	"Ru": 401,
	"Tim": 402,
	"Tap": 403,
	"Ua": 404,
	"Wa": 405,
	"Xu": 406,
	"aÃŃ": 407,
	"bo": 408,
	"boca": 409,
	"ce": 410,
	"cl": 411,
	"cÃ©": 412,
	"cÃª": 413,
	"eq": 414,
	"enga": 415,
	"erÃª": 416,
	"guara": 417,
	"ia": 418,
	"ita": 419,
	"jÃ©": 420,
	"man": 421,
	"mana": 422,
	"nd": 423,
	"pauÃ¡": 424,
	"saÃŃ": 425,
	"tÃ£": 426,
	"uca": 427,
	"uri": 428,
	"uet": 429,
	"yu": 430,
	"Ãµ": 431,
	"Ã¼": 432,
	"anhe": 433,
	"anbiquara": 434,
	"bara": 435,
	"baquara": 436,
	"uar": 437,
	"Ã©m": 438,
	"Ã§ara": 439,
	"runa": 440,
	"Cai": 441,
	"Camu": 442,
	"Arap": 443,
	"Arara": 444,
	"tir": 445,
	"tiba": 446,
	"Ama": 447,
	"Amand": 448,
	"Abaet": 449,
	"Jacu": 450,
	"Java": 451,
	"auim": 452,
	"urui": 453,
	"mbeba": 454,
	"Anh": 455,
	"Tiju": 456,
	"Tiyu": 457,
	"buna": 458,
	"uia": 459,
	"Aju": 460,
	"ParaÃŃba": 461,
	"angÃ¼": 462,
	"GuarÃ¡": 463,
	"Guarani": 464,
	"embÃ©": 465
	},
	"merges": [
	[
	"r",
	"a"
	],
	[
	"a",
	"n"
	],
	[
	"Ã",
	"¡"
	],
	[
	"b",
	"a"
	],
	[
	"c",
	"a"
	],
	[
	"u",
	"a"
	],
	[
	"Ã",
	"©"
	],
	[
	"Ã",
	"§"
	],
	[
	"r",
	"u"
	],
	[
	"an",
	"a"
	],
	[
	"C",
	"a"
	],
	[
	"i",
	"m"
	],
	[
	"A",
	"ra"
	],
	[
	"n",
	"a"
	],
	[
	"r",
	"i"
	],
	[
	"t",
	"i"
	],
	[
	"a",
	"p"
	],
	[
	"Ã",
	"ª"
	],
	[
	"Ã",
	"Ń"
	],
	[
	"A",
	"m"
	],
	[
	"a",
	"ra"
	],
	[
	"Ã",
	"£"
	],
	[
	"A",
	"ba"
	],
	[
	"Ġ",
	"("
	],
	[
	"J",
	"a"
	],
	[
	"ua",
	"ra"
	],
	[
	"a",
	"u"
	],
	[
	"r",
	"Ã©"
	],
	[
	"u",
	"ru"
	],
	[
	"m",
	"b"
	],
	[
	"n",
	"g"
	],
	[
	"r",
	"Ã¡"
	],
	[
	"Ã§",
	"a"
	],
	[
	"A",
	"n"
	],
	[
	"T",
	"i"
	],
	[
	"i",
	"ra"
	],
	[
	"m",
	"a"
	],
	[
	"t",
	"a"
	],
	[
	"K",
	"a"
	],
	[
	"b",
	"i"
	],
	[
	"b",
	"u"
	],
	[
	"c",
	"u"
	],
	[
	"n",
	"i"
	],
	[
	"u",
	"i"
	],
	[
	"v",
	"a"
	],
	[
	"w",
	"a"
	],
	[
	"A",
	"na"
	],
	[
	"b",
	"Ã¡"
	],
	[
	"e",
	"t"
	],
	[
	"h",
	"e"
	],
	[
	"ap",
	"i"
	],
	[
	"ng",
	"a"
	],
	[
	"A",
	"i"
	],
	[
	"A",
	"j"
	],
	[
	"J",
	"u"
	],
	[
	"M",
	"a"
	],
	[
	"M",
	"an"
	],
	[
	"P",
	"ara"
	],
	[
	"e",
	"ba"
	],
	[
	"g",
	"ua"
	],
	[
	"m",
	"im"
	],
	[
	"o",
	"ca"
	],
	[
	"q",
	"uara"
	],
	[
	"u",
	"Ã£"
	],
	[
	"Ã",
	"º"
	],
	[
	"Ca",
	"ri"
	],
	[
	"Am",
	"ana"
	],
	[
	"A",
	"p"
	],
	[
	"A",
	"ca"
	],
	[
	"A",
	"im"
	],
	[
	"I",
	"ra"
	],
	[
	"I",
	"ta"
	],
	[
	"J",
	"uru"
	],
	[
	"Y",
	"a"
	],
	[
	"r",
	"Ãª"
	],
	[
	"s",
	"s"
	],
	[
	"u",
	"e"
	],
	[
	"u",
	"Ã§"
	],
	[
	"x",
	"ana"
	],
	[
	"an",
	"d"
	],
	[
	"an",
	"g"
	],
	[
	"an",
	"Ã£"
	],
	[
	"Ara",
	"c"
	],
	[
	"au",
	"Ã¡"
	],
	[
	"Ã§a",
	"ÃŃ"
	],
	[
	"api",
	"xana"
	],
	[
	"A",
	"b"
	],
	[
	"A",
	"y"
	],
	[
	"A",
	"wa"
	],
	[
	"G",
	"ua"
	],
	[
	"G",
	"uara"
	],
	[
	"N",
	"he"
	],
	[
	"O",
	"ca"
	],
	[
	"P",
	"e"
	],
	[
	"P",
	"i"
	],
	[
	"T",
	"u"
	],
	[
	"X",
	"im"
	],
	[
	"a",
	"rÃ¡"
	],
	[
	"c",
	"Ã¡"
	],
	[
	"d",
	"i"
	],
	[
	"e",
	"ru"
	],
	[
	"e",
	"mb"
	],
	[
	"i",
	"t"
	],
	[
	"j",
	"u"
	],
	[
	"k",
	"a"
	],
	[
	"m",
	"u"
	],
	[
	"m",
	"ÃŃ"
	],
	[
	"o",
	"ti"
	],
	[
	"p",
	"i"
	],
	[
	"p",
	"o"
	],
	[
	"r",
	"y"
	],
	[
	"t",
	"Ãª"
	],
	[
	"u",
	"Ã§a"
	],
	[
	"Ã",
	"³"
	],
	[
	"ca",
	"mim"
	],
	[
	"Ã§",
	"u"
	],
	[
	"ÃŃ",
	"ba"
	],
	[
	"Aba",
	"ca"
	],
	[
	"bi",
	"quara"
	],
	[
	"bu",
	"rÃ©"
	],
	[
	"Ma",
	"ca"
	],
	[
	"Man",
	"i"
	],
	[
	"gua",
	"r"
	],
	[
	"Ãº",
	"na"
	],
	[
	"Tu",
	"pi"
	],
	[
	"A",
	"v"
	],
	[
	"A",
	"ti"
	],
	[
	"A",
	"va"
	],
	[
	"D",
	"a"
	],
	[
	"I",
	"a"
	],
	[
	"I",
	"ba"
	],
	[
	"I",
	"bi"
	],
	[
	"J",
	"ap"
	],
	[
	"L",
	"au"
	],
	[
	"M",
	"u"
	],
	[
	"M",
	"ana"
	],
	[
	"P",
	"uÃ§"
	],
	[
	"P",
	"arÃ¡"
	],
	[
	"P",
	"oti"
	],
	[
	"Q",
	"ui"
	],
	[
	"R",
	"u"
	],
	[
	"T",
	"im"
	],
	[
	"T",
	"ap"
	],
	[
	"U",
	"a"
	],
	[
	"W",
	"a"
	],
	[
	"X",
	"u"
	],
	[
	"a",
	"ÃŃ"
	],
	[
	"b",
	"o"
	],
	[
	"b",
	"oca"
	],
	[
	"c",
	"e"
	],
	[
	"c",
	"l"
	],
	[
	"c",
	"Ã©"
	],
	[
	"c",
	"Ãª"
	],
	[
	"e",
	"q"
	],
	[
	"e",
	"nga"
	],
	[
	"e",
	"rÃª"
	],
	[
	"g",
	"uara"
	],
	[
	"i",
	"a"
	],
	[
	"i",
	"ta"
	],
	[
	"j",
	"Ã©"
	],
	[
	"m",
	"an"
	],
	[
	"m",
	"ana"
	],
	[
	"n",
	"d"
	],
	[
	"p",
	"auÃ¡"
	],
	[
	"s",
	"aÃŃ"
	],
	[
	"t",
	"Ã£"
	],
	[
	"u",
	"ca"
	],
	[
	"u",
	"ri"
	],
	[
	"u",
	"et"
	],
	[
	"y",
	"u"
	],
	[
	"Ã",
	"µ"
	],
	[
	"Ã",
	"¼"
	],
	[
	"an",
	"he"
	],
	[
	"an",
	"biquara"
	],
	[
	"ba",
	"ra"
	],
	[
	"ba",
	"quara"
	],
	[
	"ua",
	"r"
	],
	[
	"Ã©",
	"m"
	],
	[
	"Ã§",
	"ara"
	],
	[
	"ru",
	"na"
	],
	[
	"Ca",
	"i"
	],
	[
	"Ca",
	"mu"
	],
	[
	"Ara",
	"p"
	],
	[
	"Ara",
	"ra"
	],
	[
	"ti",
	"r"
	],
	[
	"ti",
	"ba"
	],
	[
	"Am",
	"a"
	],
	[
	"Am",
	"and"
	],
	[
	"Aba",
	"et"
	],
	[
	"Ja",
	"cu"
	],
	[
	"Ja",
	"va"
	],
	[
	"au",
	"im"
	],
	[
	"uru",
	"i"
	],
	[
	"mb",
	"eba"
	],
	[
	"An",
	"h"
	],
	[
	"Ti",
	"ju"
	],
	[
	"Ti",
	"yu"
	],
	[
	"bu",
	"na"
	],
	[
	"ui",
	"a"
	],
	[
	"Aj",
	"u"
	],
	[
	"Para",
	"ÃŃba"
	],
	[
	"ang",
	"Ã¼"
	],
	[
	"Gua",
	"rÃ¡"
	],
	[
	"Guara",
	"ni"
	],
	[
	"emb",
	"Ã©"
	]
	]
	}
	}