Upload tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+text_tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

morph_tokenizer/tokenizer_config.json

@@ -0,0 +1,698 @@
+{
+  "morph_encodings": {
+    "<pad>": 0,
+    "<eos>": 1,
+    "<unk>": 2,
+    "<extra_id_0>": 3,
+    "Adj": 4,
+    "Adj-AFP": 5,
+    "Adj-AFP-C": 6,
+    "Adj-AFS": 7,
+    "Adj-AFS-C": 8,
+    "Adj-AFS-S": 9,
+    "Adj-AMP": 10,
+    "Adj-AMP-C": 11,
+    "Adj-AMS": 12,
+    "Adj-AMS-C": 13,
+    "Adj-ANP": 14,
+    "Adj-ANP-C": 15,
+    "Adj-ANP-S": 16,
+    "Adj-ANS": 17,
+    "Adj-ANS-C": 18,
+    "Adj-ANS-S": 19,
+    "Adj-DFP": 20,
+    "Adj-DFP-C": 21,
+    "Adj-DFS": 22,
+    "Adj-DFS-C": 23,
+    "Adj-DFS-S": 24,
+    "Adj-DMP": 25,
+    "Adj-DMP-C": 26,
+    "Adj-DMS": 27,
+    "Adj-DMS-C": 28,
+    "Adj-DMS-S": 29,
+    "Adj-DNP": 30,
+    "Adj-DNP-S": 31,
+    "Adj-DNS": 32,
+    "Adj-DNS-C": 33,
+    "Adj-DNS-S": 34,
+    "Adj-GFP": 35,
+    "Adj-GFP-S": 36,
+    "Adj-GFS": 37,
+    "Adj-GFS-C": 38,
+    "Adj-GMP": 39,
+    "Adj-GMP-C": 40,
+    "Adj-GMP-S": 41,
+    "Adj-GMS": 42,
+    "Adj-GMS-C": 43,
+    "Adj-GMS-S": 44,
+    "Adj-GNP": 45,
+    "Adj-GNP-C": 46,
+    "Adj-GNP-S": 47,
+    "Adj-GNS": 48,
+    "Adj-GNS-S": 49,
+    "Adj-NFP": 50,
+    "Adj-NFP-C": 51,
+    "Adj-NFP-S": 52,
+    "Adj-NFS": 53,
+    "Adj-NFS-C": 54,
+    "Adj-NFS-S": 55,
+    "Adj-NMP": 56,
+    "Adj-NMP-C": 57,
+    "Adj-NMS": 58,
+    "Adj-NMS-C": 59,
+    "Adj-NMS-S": 60,
+    "Adj-NNP": 61,
+    "Adj-NNP-C": 62,
+    "Adj-NNS": 63,
+    "Adj-NNS-C": 64,
+    "Adj-VFS": 65,
+    "Adj-VMP": 66,
+    "Adj-VMP-C": 67,
+    "Adj-VMS": 68,
+    "Adj-VMS-S": 69,
+    "Adj-VNP": 70,
+    "Adj-VNS": 71,
+    "Adv": 72,
+    "Adv-C": 73,
+    "Adv-S": 74,
+    "Art-AFP": 75,
+    "Art-AFS": 76,
+    "Art-AMP": 77,
+    "Art-AMS": 78,
+    "Art-ANP": 79,
+    "Art-ANS": 80,
+    "Art-DFP": 81,
+    "Art-DFS": 82,
+    "Art-DMP": 83,
+    "Art-DMS": 84,
+    "Art-DNP": 85,
+    "Art-DNS": 86,
+    "Art-GFP": 87,
+    "Art-GFS": 88,
+    "Art-GMP": 89,
+    "Art-GMS": 90,
+    "Art-GNP": 91,
+    "Art-GNS": 92,
+    "Art-NFP": 93,
+    "Art-NFS": 94,
+    "Art-NMP": 95,
+    "Art-NMS": 96,
+    "Art-NNP": 97,
+    "Art-NNS": 98,
+    "Art-VFP": 99,
+    "Art-VFS": 100,
+    "Art-VMP": 101,
+    "Art-VMS": 102,
+    "Art-VNP": 103,
+    "Art-VNS": 104,
+    "Conj": 105,
+    "DPro-AFP": 106,
+    "DPro-AFS": 107,
+    "DPro-AMP": 108,
+    "DPro-AMS": 109,
+    "DPro-ANP": 110,
+    "DPro-ANS": 111,
+    "DPro-DFP": 112,
+    "DPro-DFS": 113,
+    "DPro-DMP": 114,
+    "DPro-DMS": 115,
+    "DPro-DNP": 116,
+    "DPro-DNS": 117,
+    "DPro-GFP": 118,
+    "DPro-GFS": 119,
+    "DPro-GMP": 120,
+    "DPro-GMS": 121,
+    "DPro-GNP": 122,
+    "DPro-GNS": 123,
+    "DPro-NFP": 124,
+    "DPro-NFS": 125,
+    "DPro-NMP": 126,
+    "DPro-NMS": 127,
+    "DPro-NNP": 128,
+    "DPro-NNS": 129,
+    "Heb": 130,
+    "I": 131,
+    "IPro-AFP": 132,
+    "IPro-AFS": 133,
+    "IPro-AMP": 134,
+    "IPro-AMS": 135,
+    "IPro-ANP": 136,
+    "IPro-ANS": 137,
+    "IPro-DFS": 138,
+    "IPro-DMP": 139,
+    "IPro-DMS": 140,
+    "IPro-DNS": 141,
+    "IPro-GFP": 142,
+    "IPro-GFS": 143,
+    "IPro-GMP": 144,
+    "IPro-GMS": 145,
+    "IPro-GNP": 146,
+    "IPro-GNS": 147,
+    "IPro-NFP": 148,
+    "IPro-NFS": 149,
+    "IPro-NMP": 150,
+    "IPro-NMS": 151,
+    "IPro-NNP": 152,
+    "IPro-NNS": 153,
+    "Indec": 154,
+    "IntPrtcl": 155,
+    "N": 156,
+    "N-AFP": 157,
+    "N-AFS": 158,
+    "N-AMP": 159,
+    "N-AMS": 160,
+    "N-ANP": 161,
+    "N-ANS": 162,
+    "N-DFP": 163,
+    "N-DFS": 164,
+    "N-DMP": 165,
+    "N-DMS": 166,
+    "N-DNP": 167,
+    "N-DNS": 168,
+    "N-GFP": 169,
+    "N-GFS": 170,
+    "N-GMP": 171,
+    "N-GMS": 172,
+    "N-GNP": 173,
+    "N-GNS": 174,
+    "N-NFP": 175,
+    "N-NFS": 176,
+    "N-NMP": 177,
+    "N-NMS": 178,
+    "N-NNP": 179,
+    "N-NNS": 180,
+    "N-VFP": 181,
+    "N-VFS": 182,
+    "N-VMP": 183,
+    "N-VMS": 184,
+    "N-VNP": 185,
+    "N-VNS": 186,
+    "PPro-A1P": 187,
+    "PPro-A1S": 188,
+    "PPro-A2P": 189,
+    "PPro-A2S": 190,
+    "PPro-AF1P": 191,
+    "PPro-AF1S": 192,
+    "PPro-AF2P": 193,
+    "PPro-AF2S": 194,
+    "PPro-AF3P": 195,
+    "PPro-AF3S": 196,
+    "PPro-AM1P": 197,
+    "PPro-AM1S": 198,
+    "PPro-AM2P": 199,
+    "PPro-AM2S": 200,
+    "PPro-AM3P": 201,
+    "PPro-AM3S": 202,
+    "PPro-AN1P": 203,
+    "PPro-AN1S": 204,
+    "PPro-AN2P": 205,
+    "PPro-AN2S": 206,
+    "PPro-AN3P": 207,
+    "PPro-AN3S": 208,
+    "PPro-D1P": 209,
+    "PPro-D1S": 210,
+    "PPro-D2P": 211,
+    "PPro-D2S": 212,
+    "PPro-DF1P": 213,
+    "PPro-DF1S": 214,
+    "PPro-DF2P": 215,
+    "PPro-DF2S": 216,
+    "PPro-DF3P": 217,
+    "PPro-DF3S": 218,
+    "PPro-DM1P": 219,
+    "PPro-DM1S": 220,
+    "PPro-DM2P": 221,
+    "PPro-DM2S": 222,
+    "PPro-DM3P": 223,
+    "PPro-DM3S": 224,
+    "PPro-DN1P": 225,
+    "PPro-DN1S": 226,
+    "PPro-DN3P": 227,
+    "PPro-DN3S": 228,
+    "PPro-G1P": 229,
+    "PPro-G1S": 230,
+    "PPro-G2P": 231,
+    "PPro-G2S": 232,
+    "PPro-GF1P": 233,
+    "PPro-GF1S": 234,
+    "PPro-GF2P": 235,
+    "PPro-GF2S": 236,
+    "PPro-GF3P": 237,
+    "PPro-GF3S": 238,
+    "PPro-GM1S": 239,
+    "PPro-GM2S": 240,
+    "PPro-GM3P": 241,
+    "PPro-GM3S": 242,
+    "PPro-GN1P": 243,
+    "PPro-GN3P": 244,
+    "PPro-GN3S": 245,
+    "PPro-N1P": 246,
+    "PPro-N1S": 247,
+    "PPro-N2P": 248,
+    "PPro-N2S": 249,
+    "PPro-NF1P": 250,
+    "PPro-NF1S": 251,
+    "PPro-NF2P": 252,
+    "PPro-NF3S": 253,
+    "PPro-NFS": 254,
+    "PPro-NM1P": 255,
+    "PPro-NM1S": 256,
+    "PPro-NM2P": 257,
+    "PPro-NM2S": 258,
+    "PPro-NM3P": 259,
+    "PPro-NM3S": 260,
+    "PPro-NN1P": 261,
+    "PPro-NN1S": 262,
+    "PPro-NN2P": 263,
+    "PPro-NN2S": 264,
+    "PPro-NN3P": 265,
+    "PPro-NN3S": 266,
+    "Prep": 267,
+    "Prtcl": 268,
+    "RecPro-AMP": 269,
+    "RecPro-DMP": 270,
+    "RecPro-DNP": 271,
+    "RecPro-GMP": 272,
+    "RefPro-AF3P": 273,
+    "RefPro-AF3S": 274,
+    "RefPro-AM3P": 275,
+    "RefPro-AM3S": 276,
+    "RefPro-AN3P": 277,
+    "RefPro-DF3P": 278,
+    "RefPro-DF3S": 279,
+    "RefPro-DM3P": 280,
+    "RefPro-DM3S": 281,
+    "RefPro-GF3P": 282,
+    "RefPro-GF3S": 283,
+    "RefPro-GM3P": 284,
+    "RefPro-GM3S": 285,
+    "RefPro-GN3P": 286,
+    "RefPro-GN3S": 287,
+    "RelPro-AFP": 288,
+    "RelPro-AFS": 289,
+    "RelPro-AMP": 290,
+    "RelPro-AMS": 291,
+    "RelPro-ANP": 292,
+    "RelPro-ANS": 293,
+    "RelPro-DFP": 294,
+    "RelPro-DFS": 295,
+    "RelPro-DMP": 296,
+    "RelPro-DMS": 297,
+    "RelPro-DNP": 298,
+    "RelPro-DNS": 299,
+    "RelPro-GFP": 300,
+    "RelPro-GFS": 301,
+    "RelPro-GMP": 302,
+    "RelPro-GMS": 303,
+    "RelPro-GNP": 304,
+    "RelPro-GNS": 305,
+    "RelPro-NFP": 306,
+    "RelPro-NFS": 307,
+    "RelPro-NMP": 308,
+    "RelPro-NMS": 309,
+    "RelPro-NNP": 310,
+    "RelPro-NNS": 311,
+    "V-AIA-1P": 312,
+    "V-AIA-1S": 313,
+    "V-AIA-2P": 314,
+    "V-AIA-2S": 315,
+    "V-AIA-3P": 316,
+    "V-AIA-3S": 317,
+    "V-AIM-1P": 318,
+    "V-AIM-1S": 319,
+    "V-AIM-2P": 320,
+    "V-AIM-2S": 321,
+    "V-AIM-3P": 322,
+    "V-AIM-3S": 323,
+    "V-AIP-1P": 324,
+    "V-AIP-1S": 325,
+    "V-AIP-2P": 326,
+    "V-AIP-2S": 327,
+    "V-AIP-3P": 328,
+    "V-AIP-3S": 329,
+    "V-AMA-2P": 330,
+    "V-AMA-2S": 331,
+    "V-AMA-3P": 332,
+    "V-AMA-3S": 333,
+    "V-AMM-2P": 334,
+    "V-AMM-2S": 335,
+    "V-AMM-3P": 336,
+    "V-AMM-3S": 337,
+    "V-AMP-2P": 338,
+    "V-AMP-2S": 339,
+    "V-AMP-3P": 340,
+    "V-AMP-3S": 341,
+    "V-ANA": 342,
+    "V-ANM": 343,
+    "V-ANM/P": 344,
+    "V-ANP": 345,
+    "V-AOA-3P": 346,
+    "V-AOA-3S": 347,
+    "V-AOM-1S": 348,
+    "V-AOM-3S": 349,
+    "V-AOP-3S": 350,
+    "V-APA-AFP": 351,
+    "V-APA-AFS": 352,
+    "V-APA-AMP": 353,
+    "V-APA-AMS": 354,
+    "V-APA-ANP": 355,
+    "V-APA-ANS": 356,
+    "V-APA-DFP": 357,
+    "V-APA-DFS": 358,
+    "V-APA-DMP": 359,
+    "V-APA-DMS": 360,
+    "V-APA-GFS": 361,
+    "V-APA-GMP": 362,
+    "V-APA-GMS": 363,
+    "V-APA-GNS": 364,
+    "V-APA-NFP": 365,
+    "V-APA-NFS": 366,
+    "V-APA-NMP": 367,
+    "V-APA-NMS": 368,
+    "V-APA-NNP": 369,
+    "V-APA-NNS": 370,
+    "V-APM-AFS": 371,
+    "V-APM-AMP": 372,
+    "V-APM-AMS": 373,
+    "V-APM-ANP": 374,
+    "V-APM-ANS": 375,
+    "V-APM-DMP": 376,
+    "V-APM-DNP": 377,
+    "V-APM-GFP": 378,
+    "V-APM-GFS": 379,
+    "V-APM-GMP": 380,
+    "V-APM-GMS": 381,
+    "V-APM-GNP": 382,
+    "V-APM-GNS": 383,
+    "V-APM-NFP": 384,
+    "V-APM-NFS": 385,
+    "V-APM-NMP": 386,
+    "V-APM-NMS": 387,
+    "V-APM-NNS": 388,
+    "V-APM/P-ANP": 389,
+    "V-APM/P-GFS": 390,
+    "V-APM/P-NMS": 391,
+    "V-APP-AFS": 392,
+    "V-APP-AMP": 393,
+    "V-APP-AMS": 394,
+    "V-APP-ANP": 395,
+    "V-APP-ANS": 396,
+    "V-APP-DFS": 397,
+    "V-APP-DMS": 398,
+    "V-APP-DNP": 399,
+    "V-APP-DNS": 400,
+    "V-APP-GFP": 401,
+    "V-APP-GFS": 402,
+    "V-APP-GMP": 403,
+    "V-APP-GMS": 404,
+    "V-APP-GNP": 405,
+    "V-APP-GNS": 406,
+    "V-APP-NFP": 407,
+    "V-APP-NFS": 408,
+    "V-APP-NMP": 409,
+    "V-APP-NMS": 410,
+    "V-APP-NNP": 411,
+    "V-APP-NNS": 412,
+    "V-ASA-1P": 413,
+    "V-ASA-1S": 414,
+    "V-ASA-2P": 415,
+    "V-ASA-2S": 416,
+    "V-ASA-3P": 417,
+    "V-ASA-3S": 418,
+    "V-ASM-1P": 419,
+    "V-ASM-1S": 420,
+    "V-ASM-2P": 421,
+    "V-ASM-2S": 422,
+    "V-ASM-3P": 423,
+    "V-ASM-3S": 424,
+    "V-ASP-1P": 425,
+    "V-ASP-1S": 426,
+    "V-ASP-2P": 427,
+    "V-ASP-2S": 428,
+    "V-ASP-3P": 429,
+    "V-ASP-3S": 430,
+    "V-FIA-1P": 431,
+    "V-FIA-1S": 432,
+    "V-FIA-2P": 433,
+    "V-FIA-2S": 434,
+    "V-FIA-3P": 435,
+    "V-FIA-3S": 436,
+    "V-FIM-1P": 437,
+    "V-FIM-1S": 438,
+    "V-FIM-2P": 439,
+    "V-FIM-2S": 440,
+    "V-FIM-3P": 441,
+    "V-FIM-3S": 442,
+    "V-FIM/P-2P": 443,
+    "V-FIP-1P": 444,
+    "V-FIP-1S": 445,
+    "V-FIP-2P": 446,
+    "V-FIP-2S": 447,
+    "V-FIP-3P": 448,
+    "V-FIP-3S": 449,
+    "V-FNM": 450,
+    "V-FPA-ANP": 451,
+    "V-FPA-NMP": 452,
+    "V-FPA-NMS": 453,
+    "V-FPM-ANS": 454,
+    "V-FPP-GNP": 455,
+    "V-IIA-1P": 456,
+    "V-IIA-1S": 457,
+    "V-IIA-2P": 458,
+    "V-IIA-2S": 459,
+    "V-IIA-3P": 460,
+    "V-IIA-3S": 461,
+    "V-IIM-1P": 462,
+    "V-IIM-1S": 463,
+    "V-IIM-2S": 464,
+    "V-IIM-3P": 465,
+    "V-IIM-3S": 466,
+    "V-IIM/P-1P": 467,
+    "V-IIM/P-1S": 468,
+    "V-IIM/P-2P": 469,
+    "V-IIM/P-2S": 470,
+    "V-IIM/P-3P": 471,
+    "V-IIM/P-3S": 472,
+    "V-IIP-1P": 473,
+    "V-IIP-3P": 474,
+    "V-LIA-1S": 475,
+    "V-LIA-2P": 476,
+    "V-LIA-2S": 477,
+    "V-LIA-3P": 478,
+    "V-LIA-3S": 479,
+    "V-LIM-3P": 480,
+    "V-LIM-3S": 481,
+    "V-LIM/P-3S": 482,
+    "V-M-2P": 483,
+    "V-M-2S": 484,
+    "V-PIA-1P": 485,
+    "V-PIA-1S": 486,
+    "V-PIA-2P": 487,
+    "V-PIA-2S": 488,
+    "V-PIA-3P": 489,
+    "V-PIA-3S": 490,
+    "V-PIM-1P": 491,
+    "V-PIM-1S": 492,
+    "V-PIM-2P": 493,
+    "V-PIM-3P": 494,
+    "V-PIM-3S": 495,
+    "V-PIM/P-1P": 496,
+    "V-PIM/P-1S": 497,
+    "V-PIM/P-2P": 498,
+    "V-PIM/P-2S": 499,
+    "V-PIM/P-3P": 500,
+    "V-PIM/P-3S": 501,
+    "V-PIP-1S": 502,
+    "V-PIP-3S": 503,
+    "V-PMA-2P": 504,
+    "V-PMA-2S": 505,
+    "V-PMA-3P": 506,
+    "V-PMA-3S": 507,
+    "V-PMM-2P": 508,
+    "V-PMM-2S": 509,
+    "V-PMM/P-2P": 510,
+    "V-PMM/P-2S": 511,
+    "V-PMM/P-3P": 512,
+    "V-PMM/P-3S": 513,
+    "V-PMP-2P": 514,
+    "V-PMP-3S": 515,
+    "V-PNA": 516,
+    "V-PNM": 517,
+    "V-PNM/P": 518,
+    "V-PNP": 519,
+    "V-POA-2P": 520,
+    "V-POA-3P": 521,
+    "V-POA-3S": 522,
+    "V-POM/P-1S": 523,
+    "V-POM/P-3P": 524,
+    "V-POM/P-3S": 525,
+    "V-PPA-AFP": 526,
+    "V-PPA-AFS": 527,
+    "V-PPA-AMP": 528,
+    "V-PPA-AMS": 529,
+    "V-PPA-ANP": 530,
+    "V-PPA-ANS": 531,
+    "V-PPA-DFP": 532,
+    "V-PPA-DFS": 533,
+    "V-PPA-DMP": 534,
+    "V-PPA-DMS": 535,
+    "V-PPA-DNP": 536,
+    "V-PPA-DNS": 537,
+    "V-PPA-GFP": 538,
+    "V-PPA-GFS": 539,
+    "V-PPA-GMP": 540,
+    "V-PPA-GMS": 541,
+    "V-PPA-GNP": 542,
+    "V-PPA-GNS": 543,
+    "V-PPA-NFP": 544,
+    "V-PPA-NFS": 545,
+    "V-PPA-NMP": 546,
+    "V-PPA-NMS": 547,
+    "V-PPA-NNP": 548,
+    "V-PPA-NNS": 549,
+    "V-PPA-VFS": 550,
+    "V-PPA-VMP": 551,
+    "V-PPA-VMS": 552,
+    "V-PPM-AFS": 553,
+    "V-PPM-AMP": 554,
+    "V-PPM-AMS": 555,
+    "V-PPM-ANS": 556,
+    "V-PPM-DMP": 557,
+    "V-PPM-DMS": 558,
+    "V-PPM-GFP": 559,
+    "V-PPM-GFS": 560,
+    "V-PPM-GMP": 561,
+    "V-PPM-GMS": 562,
+    "V-PPM-GNP": 563,
+    "V-PPM-GNS": 564,
+    "V-PPM-NFP": 565,
+    "V-PPM-NFS": 566,
+    "V-PPM-NMP": 567,
+    "V-PPM-NMS": 568,
+    "V-PPM-NNS": 569,
+    "V-PPM/P-AFP": 570,
+    "V-PPM/P-AFS": 571,
+    "V-PPM/P-AMP": 572,
+    "V-PPM/P-AMS": 573,
+    "V-PPM/P-ANP": 574,
+    "V-PPM/P-ANS": 575,
+    "V-PPM/P-DFP": 576,
+    "V-PPM/P-DFS": 577,
+    "V-PPM/P-DMP": 578,
+    "V-PPM/P-DMS": 579,
+    "V-PPM/P-DNP": 580,
+    "V-PPM/P-DNS": 581,
+    "V-PPM/P-GFP": 582,
+    "V-PPM/P-GFS": 583,
+    "V-PPM/P-GMP": 584,
+    "V-PPM/P-GMS": 585,
+    "V-PPM/P-GNP": 586,
+    "V-PPM/P-GNS": 587,
+    "V-PPM/P-NFP": 588,
+    "V-PPM/P-NFS": 589,
+    "V-PPM/P-NMP": 590,
+    "V-PPM/P-NMS": 591,
+    "V-PPM/P-NNP": 592,
+    "V-PPM/P-NNS": 593,
+    "V-PPM/P-VMP": 594,
+    "V-PPM/P-VMS": 595,
+    "V-PPP-DMP": 596,
+    "V-PPP-GMP": 597,
+    "V-PPP-GMS": 598,
+    "V-PPP-NMP": 599,
+    "V-PSA-1P": 600,
+    "V-PSA-1S": 601,
+    "V-PSA-2P": 602,
+    "V-PSA-2S": 603,
+    "V-PSA-3P": 604,
+    "V-PSA-3S": 605,
+    "V-PSM-1P": 606,
+    "V-PSM-1S": 607,
+    "V-PSM-2S": 608,
+    "V-PSM-3S": 609,
+    "V-PSM/P-1P": 610,
+    "V-PSM/P-1S": 611,
+    "V-PSM/P-2P": 612,
+    "V-PSM/P-2S": 613,
+    "V-PSM/P-3P": 614,
+    "V-PSM/P-3S": 615,
+    "V-RIA-1P": 616,
+    "V-RIA-1S": 617,
+    "V-RIA-2P": 618,
+    "V-RIA-2S": 619,
+    "V-RIA-3P": 620,
+    "V-RIA-3S": 621,
+    "V-RIM-2S": 622,
+    "V-RIM-3S": 623,
+    "V-RIM/P-1P": 624,
+    "V-RIM/P-1S": 625,
+    "V-RIM/P-2P": 626,
+    "V-RIM/P-2S": 627,
+    "V-RIM/P-3P": 628,
+    "V-RIM/P-3S": 629,
+    "V-RIP-1P": 630,
+    "V-RMA-2P": 631,
+    "V-RMM/P-2P": 632,
+    "V-RMM/P-2S": 633,
+    "V-RNA": 634,
+    "V-RNM/P": 635,
+    "V-RPA-AFS": 636,
+    "V-RPA-AMP": 637,
+    "V-RPA-AMS": 638,
+    "V-RPA-ANP": 639,
+    "V-RPA-ANS": 640,
+    "V-RPA-DMP": 641,
+    "V-RPA-DMS": 642,
+    "V-RPA-DNS": 643,
+    "V-RPA-GFS": 644,
+    "V-RPA-GMP": 645,
+    "V-RPA-GMS": 646,
+    "V-RPA-GNP": 647,
+    "V-RPA-NFP": 648,
+    "V-RPA-NFS": 649,
+    "V-RPA-NMP": 650,
+    "V-RPA-NMS": 651,
+    "V-RPA-NNP": 652,
+    "V-RPA-NNS": 653,
+    "V-RPM-AMS": 654,
+    "V-RPM-NMP": 655,
+    "V-RPM-NMS": 656,
+    "V-RPM/P-AFP": 657,
+    "V-RPM/P-AFS": 658,
+    "V-RPM/P-AMP": 659,
+    "V-RPM/P-AMS": 660,
+    "V-RPM/P-ANP": 661,
+    "V-RPM/P-ANS": 662,
+    "V-RPM/P-DFP": 663,
+    "V-RPM/P-DFS": 664,
+    "V-RPM/P-DMP": 665,
+    "V-RPM/P-DMS": 666,
+    "V-RPM/P-DNP": 667,
+    "V-RPM/P-DNS": 668,
+    "V-RPM/P-GFP": 669,
+    "V-RPM/P-GFS": 670,
+    "V-RPM/P-GMP": 671,
+    "V-RPM/P-GMS": 672,
+    "V-RPM/P-GNP": 673,
+    "V-RPM/P-GNS": 674,
+    "V-RPM/P-NFP": 675,
+    "V-RPM/P-NFS": 676,
+    "V-RPM/P-NMP": 677,
+    "V-RPM/P-NMS": 678,
+    "V-RPM/P-NNP": 679,
+    "V-RPM/P-NNS": 680,
+    "V-RPM/P-VFS": 681,
+    "V-RPM/P-VMP": 682,
+    "V-RPM/P-VMS": 683,
+    "V-RSA-1P": 684,
+    "V-RSA-1S": 685,
+    "V-RSA-2P": 686,
+    "V-RSA-2S": 687
+  },
+  "special_tokens_map": {
+    "pad_token": "<pad>",
+    "eos_token": "<eos>",
+    "unk_token": "<unk>",
+    "block_separator_token": "<extra_id_0>"
+  }
+}

text_tokenizer/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<extra_id_0>": 250100,
+  "<extra_id_1>": 250101,
+  "<extra_id_2>": 250102
+}

text_tokenizer/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

text_tokenizer/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

text_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3177a3143849c3ea44b46cfd1cc08d456c5a9a21e68d0d7d9c09d5802bbcd973
+size 16315688

text_tokenizer/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "additional_special_tokens": null,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}