Initial upload of fine-tuned model

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-3670/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-7340/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,58 @@
+---
+base_model: Maxtimer97/Llama2Hymba_raw
+library_name: transformers
+model_name: model
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for model
+
+This model is a fine-tuned version of [Maxtimer97/Llama2Hymba_raw](https://huggingface.co/Maxtimer97/Llama2Hymba_raw).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/maxtimer/huggingface/runs/0tli9m2w) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.19.1
+- Transformers: 4.53.0
+- Pytorch: 2.6.0
+- Datasets: 3.6.0
+- Tokenizers: 0.21.1
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-3670/chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-3670/config.json

@@ -0,0 +1,132 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_factor": 0.5,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": -1,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": false,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 16,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 4,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 256,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 128,
+  "vocab_size": 128256
+}

checkpoint-3670/configuration_hymba.py

@@ -0,0 +1,122 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            attn_factor=0.5,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.attn_factor = attn_factor
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+        
+        self.pure_attn = pure_attn
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

checkpoint-3670/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

checkpoint-3670/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2decd3903100797d7bee8fb369c5c169f6aee80480ac11adbec530b931b8482d
+size 2471642200

checkpoint-3670/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4436563ae95994e3dc556265a03fa939edf4ae59caa01d5c852f8cfe68369dcb
+size 4943354422

checkpoint-3670/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-3670/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c99d7c54f1bcd86b8012d20bcfb3cf68e8fc210787c8328b62c251974eeae7f
+size 1064

checkpoint-3670/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-3670/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-3670/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-3670/trainer_state.json

@@ -0,0 +1,3346 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 3670,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0002725306171115161,
+      "grad_norm": 106.0,
+      "learning_rate": 0.0,
+      "loss": 11.3973,
+      "mean_token_accuracy": 0.010489485081052408,
+      "num_tokens": 180497.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0027253061711151614,
+      "grad_norm": 57.0,
+      "learning_rate": 2.0361990950226245e-05,
+      "loss": 10.0617,
+      "mean_token_accuracy": 0.018964683349218428,
+      "num_tokens": 1772191.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.005450612342230323,
+      "grad_norm": 37.75,
+      "learning_rate": 4.2986425339366516e-05,
+      "loss": 8.1695,
+      "mean_token_accuracy": 0.04115874093695311,
+      "num_tokens": 3528128.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.008175918513345483,
+      "grad_norm": 26.25,
+      "learning_rate": 6.561085972850679e-05,
+      "loss": 6.7807,
+      "mean_token_accuracy": 0.07557539474219084,
+      "num_tokens": 5321820.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.010901224684460645,
+      "grad_norm": 20.125,
+      "learning_rate": 8.823529411764706e-05,
+      "loss": 5.4023,
+      "mean_token_accuracy": 0.1482221502577886,
+      "num_tokens": 7036353.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.013626530855575806,
+      "grad_norm": 7.53125,
+      "learning_rate": 0.00011085972850678733,
+      "loss": 4.5179,
+      "mean_token_accuracy": 0.2088393464917317,
+      "num_tokens": 8794502.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.016351837026690966,
+      "grad_norm": 8.0625,
+      "learning_rate": 0.0001334841628959276,
+      "loss": 3.8857,
+      "mean_token_accuracy": 0.26825836482457815,
+      "num_tokens": 10519144.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.019077143197806127,
+      "grad_norm": 9.3125,
+      "learning_rate": 0.00015610859728506788,
+      "loss": 3.2721,
+      "mean_token_accuracy": 0.348467530682683,
+      "num_tokens": 12272638.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.02180244936892129,
+      "grad_norm": 3.703125,
+      "learning_rate": 0.00017873303167420816,
+      "loss": 2.8456,
+      "mean_token_accuracy": 0.41965038063935933,
+      "num_tokens": 14037595.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.02452775554003645,
+      "grad_norm": 2.84375,
+      "learning_rate": 0.0002013574660633484,
+      "loss": 2.5555,
+      "mean_token_accuracy": 0.46718343761749564,
+      "num_tokens": 15813413.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.027253061711151612,
+      "grad_norm": 3.140625,
+      "learning_rate": 0.0002239819004524887,
+      "loss": 2.3813,
+      "mean_token_accuracy": 0.4977288795635104,
+      "num_tokens": 17606667.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.029978367882266772,
+      "grad_norm": 5.03125,
+      "learning_rate": 0.000246606334841629,
+      "loss": 2.1859,
+      "mean_token_accuracy": 0.5393931507132947,
+      "num_tokens": 19387650.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.03270367405338193,
+      "grad_norm": 2.125,
+      "learning_rate": 0.0002692307692307692,
+      "loss": 2.1126,
+      "mean_token_accuracy": 0.5529376215301454,
+      "num_tokens": 21111629.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.03542898022449709,
+      "grad_norm": 2.390625,
+      "learning_rate": 0.00029185520361990955,
+      "loss": 2.1077,
+      "mean_token_accuracy": 0.561456169653684,
+      "num_tokens": 22855847.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.038154286395612254,
+      "grad_norm": 2.8125,
+      "learning_rate": 0.0003144796380090498,
+      "loss": 1.9626,
+      "mean_token_accuracy": 0.5820818378590047,
+      "num_tokens": 24647294.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.04087959256672742,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.00033710407239819005,
+      "loss": 1.9295,
+      "mean_token_accuracy": 0.5896116701886058,
+      "num_tokens": 26336128.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.04360489873784258,
+      "grad_norm": 2.046875,
+      "learning_rate": 0.00035972850678733033,
+      "loss": 1.9306,
+      "mean_token_accuracy": 0.5908254419453443,
+      "num_tokens": 28061528.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.04633020490895774,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.00038235294117647055,
+      "loss": 1.9056,
+      "mean_token_accuracy": 0.5959834239445627,
+      "num_tokens": 29834651.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.0490555110800729,
+      "grad_norm": 1.6953125,
+      "learning_rate": 0.0004049773755656109,
+      "loss": 1.9277,
+      "mean_token_accuracy": 0.595614176010713,
+      "num_tokens": 31533806.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.05178081725118806,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.0004276018099547511,
+      "loss": 1.8552,
+      "mean_token_accuracy": 0.606397076882422,
+      "num_tokens": 33328644.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.054506123422303224,
+      "grad_norm": 1.6484375,
+      "learning_rate": 0.00045022624434389144,
+      "loss": 1.8461,
+      "mean_token_accuracy": 0.6106193735264241,
+      "num_tokens": 35085202.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.057231429593418384,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.00047285067873303167,
+      "loss": 1.8531,
+      "mean_token_accuracy": 0.6109982108697295,
+      "num_tokens": 36899685.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.059956735764533545,
+      "grad_norm": 1.34375,
+      "learning_rate": 0.000495475113122172,
+      "loss": 1.7533,
+      "mean_token_accuracy": 0.6274582323618233,
+      "num_tokens": 38663867.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.0626820419356487,
+      "grad_norm": 1.5625,
+      "learning_rate": 0.0004994381233319287,
+      "loss": 1.7962,
+      "mean_token_accuracy": 0.62058030301705,
+      "num_tokens": 40368688.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.06540734810676387,
+      "grad_norm": 1.234375,
+      "learning_rate": 0.0004987357774968394,
+      "loss": 1.795,
+      "mean_token_accuracy": 0.6195226193405687,
+      "num_tokens": 42091556.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.06813265427787903,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.0004980334316617502,
+      "loss": 1.7885,
+      "mean_token_accuracy": 0.6225197708234191,
+      "num_tokens": 43832865.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.07085796044899419,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.000497331085826661,
+      "loss": 1.777,
+      "mean_token_accuracy": 0.6262112149037421,
+      "num_tokens": 45556421.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.07358326662010935,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.0004966287399915719,
+      "loss": 1.8368,
+      "mean_token_accuracy": 0.6205988208763301,
+      "num_tokens": 47284838.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.07630857279122451,
+      "grad_norm": 1.2734375,
+      "learning_rate": 0.0004959263941564827,
+      "loss": 1.7812,
+      "mean_token_accuracy": 0.6273719569668174,
+      "num_tokens": 49025592.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.07903387896233968,
+      "grad_norm": 1.1875,
+      "learning_rate": 0.0004952240483213935,
+      "loss": 1.8823,
+      "mean_token_accuracy": 0.613296614214778,
+      "num_tokens": 50771353.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.08175918513345484,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0004945217024863043,
+      "loss": 1.7168,
+      "mean_token_accuracy": 0.6380896210670471,
+      "num_tokens": 52521467.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.08448449130457,
+      "grad_norm": 1.1328125,
+      "learning_rate": 0.0004938193566512151,
+      "loss": 1.7443,
+      "mean_token_accuracy": 0.6318002883344889,
+      "num_tokens": 54338858.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.08720979747568516,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.0004931170108161258,
+      "loss": 1.7661,
+      "mean_token_accuracy": 0.6319120707921684,
+      "num_tokens": 56014081.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.08993510364680032,
+      "grad_norm": 1.109375,
+      "learning_rate": 0.0004924146649810366,
+      "loss": 1.7026,
+      "mean_token_accuracy": 0.6394819458015263,
+      "num_tokens": 57778337.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.09266040981791548,
+      "grad_norm": 0.9375,
+      "learning_rate": 0.0004917123191459475,
+      "loss": 1.7264,
+      "mean_token_accuracy": 0.6390960057266056,
+      "num_tokens": 59528977.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.09538571598903064,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.0004910099733108583,
+      "loss": 1.7108,
+      "mean_token_accuracy": 0.639170390740037,
+      "num_tokens": 61295985.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.0981110221601458,
+      "grad_norm": 1.109375,
+      "learning_rate": 0.0004903076274757691,
+      "loss": 1.6902,
+      "mean_token_accuracy": 0.6423498384654522,
+      "num_tokens": 63026897.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.10083632833126097,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.0004896052816406799,
+      "loss": 1.6393,
+      "mean_token_accuracy": 0.6515957163646817,
+      "num_tokens": 64770711.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.10356163450237613,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.0004889029358055907,
+      "loss": 1.6554,
+      "mean_token_accuracy": 0.6511010814458131,
+      "num_tokens": 66551008.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.10628694067349129,
+      "grad_norm": 0.734375,
+      "learning_rate": 0.0004882005899705015,
+      "loss": 1.6654,
+      "mean_token_accuracy": 0.6467505861073732,
+      "num_tokens": 68320690.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.10901224684460645,
+      "grad_norm": 0.8046875,
+      "learning_rate": 0.0004874982441354123,
+      "loss": 1.5943,
+      "mean_token_accuracy": 0.6583768699318171,
+      "num_tokens": 70095284.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.11173755301572161,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.0004867958983003231,
+      "loss": 1.6378,
+      "mean_token_accuracy": 0.65406036041677,
+      "num_tokens": 71803511.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.11446285918683677,
+      "grad_norm": 0.68359375,
+      "learning_rate": 0.0004860935524652339,
+      "loss": 1.5457,
+      "mean_token_accuracy": 0.667325837444514,
+      "num_tokens": 73528297.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.11718816535795193,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.0004853912066301447,
+      "loss": 1.5515,
+      "mean_token_accuracy": 0.6666892973706127,
+      "num_tokens": 75274289.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.11991347152906709,
+      "grad_norm": 0.875,
+      "learning_rate": 0.00048468886079505545,
+      "loss": 1.6137,
+      "mean_token_accuracy": 0.6556662306189537,
+      "num_tokens": 77077403.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.12263877770018225,
+      "grad_norm": 0.80859375,
+      "learning_rate": 0.0004839865149599663,
+      "loss": 1.5933,
+      "mean_token_accuracy": 0.6582868808880449,
+      "num_tokens": 78859882.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.1253640838712974,
+      "grad_norm": 1.5078125,
+      "learning_rate": 0.0004832841691248771,
+      "loss": 1.6189,
+      "mean_token_accuracy": 0.6552375074476003,
+      "num_tokens": 80590976.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.12808939004241257,
+      "grad_norm": 1.75,
+      "learning_rate": 0.0004825818232897879,
+      "loss": 1.5785,
+      "mean_token_accuracy": 0.6629863957874477,
+      "num_tokens": 82275126.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.13081469621352773,
+      "grad_norm": 0.6796875,
+      "learning_rate": 0.00048187947745469866,
+      "loss": 1.5519,
+      "mean_token_accuracy": 0.6660696087405086,
+      "num_tokens": 84096910.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.1335400023846429,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.00048117713161960953,
+      "loss": 1.5893,
+      "mean_token_accuracy": 0.6608347145840525,
+      "num_tokens": 85876326.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.13626530855575805,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.0004804747857845203,
+      "loss": 1.5587,
+      "mean_token_accuracy": 0.6648266627453268,
+      "num_tokens": 87607478.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.1389906147268732,
+      "grad_norm": 0.66796875,
+      "learning_rate": 0.0004797724399494311,
+      "loss": 1.4929,
+      "mean_token_accuracy": 0.6747789577580988,
+      "num_tokens": 89350066.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.14171592089798837,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.0004790700941143419,
+      "loss": 1.5088,
+      "mean_token_accuracy": 0.6725917850621045,
+      "num_tokens": 91067910.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.14444122706910353,
+      "grad_norm": 0.57421875,
+      "learning_rate": 0.00047836774827925274,
+      "loss": 1.4991,
+      "mean_token_accuracy": 0.6736135099083185,
+      "num_tokens": 92797017.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.1471665332402187,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.0004776654024441635,
+      "loss": 1.5442,
+      "mean_token_accuracy": 0.6660140948370099,
+      "num_tokens": 94606329.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.14989183941133385,
+      "grad_norm": 0.64453125,
+      "learning_rate": 0.0004769630566090743,
+      "loss": 1.4942,
+      "mean_token_accuracy": 0.6727027535438538,
+      "num_tokens": 96331087.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.15261714558244902,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.00047626071077398513,
+      "loss": 1.5207,
+      "mean_token_accuracy": 0.6709301644004881,
+      "num_tokens": 98138711.0,
+      "step": 560
+    },
+    {
+      "epoch": 0.15534245175356418,
+      "grad_norm": 1.0625,
+      "learning_rate": 0.00047555836493889595,
+      "loss": 1.5346,
+      "mean_token_accuracy": 0.6697394871152937,
+      "num_tokens": 99885005.0,
+      "step": 570
+    },
+    {
+      "epoch": 0.15806775792467936,
+      "grad_norm": 0.75390625,
+      "learning_rate": 0.0004748560191038067,
+      "loss": 1.3995,
+      "mean_token_accuracy": 0.6897108962759375,
+      "num_tokens": 101636075.0,
+      "step": 580
+    },
+    {
+      "epoch": 0.16079306409579452,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.0004741536732687175,
+      "loss": 1.4711,
+      "mean_token_accuracy": 0.6787430424243212,
+      "num_tokens": 103349118.0,
+      "step": 590
+    },
+    {
+      "epoch": 0.16351837026690969,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.00047345132743362834,
+      "loss": 1.4826,
+      "mean_token_accuracy": 0.6767816330306232,
+      "num_tokens": 105033010.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.16624367643802485,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.00047274898159853915,
+      "loss": 1.4251,
+      "mean_token_accuracy": 0.6852256084792316,
+      "num_tokens": 106723283.0,
+      "step": 610
+    },
+    {
+      "epoch": 0.16896898260914,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.0004720466357634499,
+      "loss": 1.4393,
+      "mean_token_accuracy": 0.6821009494364262,
+      "num_tokens": 108436878.0,
+      "step": 620
+    },
+    {
+      "epoch": 0.17169428878025517,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.0004713442899283608,
+      "loss": 1.4218,
+      "mean_token_accuracy": 0.6854843683540821,
+      "num_tokens": 110203157.0,
+      "step": 630
+    },
+    {
+      "epoch": 0.17441959495137033,
+      "grad_norm": 0.6484375,
+      "learning_rate": 0.00047064194409327155,
+      "loss": 1.4457,
+      "mean_token_accuracy": 0.6829626850783825,
+      "num_tokens": 111949130.0,
+      "step": 640
+    },
+    {
+      "epoch": 0.1771449011224855,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.0004699395982581823,
+      "loss": 1.4549,
+      "mean_token_accuracy": 0.6785116318613291,
+      "num_tokens": 113652926.0,
+      "step": 650
+    },
+    {
+      "epoch": 0.17987020729360065,
+      "grad_norm": 0.53125,
+      "learning_rate": 0.0004692372524230931,
+      "loss": 1.4303,
+      "mean_token_accuracy": 0.6821146721020341,
+      "num_tokens": 115334647.0,
+      "step": 660
+    },
+    {
+      "epoch": 0.1825955134647158,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.00046853490658800394,
+      "loss": 1.3843,
+      "mean_token_accuracy": 0.693703673966229,
+      "num_tokens": 117032381.0,
+      "step": 670
+    },
+    {
+      "epoch": 0.18532081963583097,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.00046783256075291475,
+      "loss": 1.4299,
+      "mean_token_accuracy": 0.6833634852431715,
+      "num_tokens": 118801553.0,
+      "step": 680
+    },
+    {
+      "epoch": 0.18804612580694613,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.0004671302149178255,
+      "loss": 1.4306,
+      "mean_token_accuracy": 0.6827819195576013,
+      "num_tokens": 120530271.0,
+      "step": 690
+    },
+    {
+      "epoch": 0.1907714319780613,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.00046642786908273633,
+      "loss": 1.4207,
+      "mean_token_accuracy": 0.6865599798038602,
+      "num_tokens": 122314411.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.19349673814917645,
+      "grad_norm": 0.474609375,
+      "learning_rate": 0.00046572552324764715,
+      "loss": 1.4071,
+      "mean_token_accuracy": 0.687783928308636,
+      "num_tokens": 124054113.0,
+      "step": 710
+    },
+    {
+      "epoch": 0.1962220443202916,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.00046502317741255796,
+      "loss": 1.4147,
+      "mean_token_accuracy": 0.6855330302380025,
+      "num_tokens": 125786705.0,
+      "step": 720
+    },
+    {
+      "epoch": 0.19894735049140677,
+      "grad_norm": 0.53125,
+      "learning_rate": 0.0004643208315774687,
+      "loss": 1.4339,
+      "mean_token_accuracy": 0.6828508110716939,
+      "num_tokens": 127510112.0,
+      "step": 730
+    },
+    {
+      "epoch": 0.20167265666252193,
+      "grad_norm": 0.490234375,
+      "learning_rate": 0.0004636184857423796,
+      "loss": 1.3693,
+      "mean_token_accuracy": 0.6937206975184381,
+      "num_tokens": 129321733.0,
+      "step": 740
+    },
+    {
+      "epoch": 0.2043979628336371,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.00046291613990729035,
+      "loss": 1.4267,
+      "mean_token_accuracy": 0.6843720972537994,
+      "num_tokens": 131068939.0,
+      "step": 750
+    },
+    {
+      "epoch": 0.20712326900475225,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.00046221379407220117,
+      "loss": 1.4647,
+      "mean_token_accuracy": 0.6786275329068303,
+      "num_tokens": 132800192.0,
+      "step": 760
+    },
+    {
+      "epoch": 0.2098485751758674,
+      "grad_norm": 0.5,
+      "learning_rate": 0.00046151144823711193,
+      "loss": 1.3739,
+      "mean_token_accuracy": 0.6946681007742882,
+      "num_tokens": 134501880.0,
+      "step": 770
+    },
+    {
+      "epoch": 0.21257388134698257,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.0004608091024020228,
+      "loss": 1.3379,
+      "mean_token_accuracy": 0.6989552618004382,
+      "num_tokens": 136227230.0,
+      "step": 780
+    },
+    {
+      "epoch": 0.21529918751809773,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.00046010675656693356,
+      "loss": 1.3646,
+      "mean_token_accuracy": 0.6950390106998384,
+      "num_tokens": 137938433.0,
+      "step": 790
+    },
+    {
+      "epoch": 0.2180244936892129,
+      "grad_norm": 0.453125,
+      "learning_rate": 0.0004594044107318444,
+      "loss": 1.3744,
+      "mean_token_accuracy": 0.6953079327940941,
+      "num_tokens": 139667163.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.22074979986032806,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.00045870206489675514,
+      "loss": 1.3347,
+      "mean_token_accuracy": 0.7004647579975426,
+      "num_tokens": 141481099.0,
+      "step": 810
+    },
+    {
+      "epoch": 0.22347510603144322,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.000457999719061666,
+      "loss": 1.4183,
+      "mean_token_accuracy": 0.6861088563688099,
+      "num_tokens": 143204243.0,
+      "step": 820
+    },
+    {
+      "epoch": 0.22620041220255838,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.00045729737322657677,
+      "loss": 1.3501,
+      "mean_token_accuracy": 0.6989492080174387,
+      "num_tokens": 144995581.0,
+      "step": 830
+    },
+    {
+      "epoch": 0.22892571837367354,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.0004565950273914876,
+      "loss": 1.3681,
+      "mean_token_accuracy": 0.6930115528404712,
+      "num_tokens": 146711076.0,
+      "step": 840
+    },
+    {
+      "epoch": 0.2316510245447887,
+      "grad_norm": 0.44921875,
+      "learning_rate": 0.0004558926815563984,
+      "loss": 1.335,
+      "mean_token_accuracy": 0.7001006077043712,
+      "num_tokens": 148463902.0,
+      "step": 850
+    },
+    {
+      "epoch": 0.23437633071590386,
+      "grad_norm": 0.53125,
+      "learning_rate": 0.0004551903357213092,
+      "loss": 1.3779,
+      "mean_token_accuracy": 0.6926328887231648,
+      "num_tokens": 150212801.0,
+      "step": 860
+    },
+    {
+      "epoch": 0.23710163688701902,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.00045448798988622,
+      "loss": 1.3138,
+      "mean_token_accuracy": 0.7013914404436946,
+      "num_tokens": 151950016.0,
+      "step": 870
+    },
+    {
+      "epoch": 0.23982694305813418,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.00045378564405113074,
+      "loss": 1.3262,
+      "mean_token_accuracy": 0.7006035029888154,
+      "num_tokens": 153686341.0,
+      "step": 880
+    },
+    {
+      "epoch": 0.24255224922924934,
+      "grad_norm": 0.46875,
+      "learning_rate": 0.0004530832982160416,
+      "loss": 1.3548,
+      "mean_token_accuracy": 0.694561337120831,
+      "num_tokens": 155442220.0,
+      "step": 890
+    },
+    {
+      "epoch": 0.2452775554003645,
+      "grad_norm": 0.44140625,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 1.3115,
+      "mean_token_accuracy": 0.703803816344589,
+      "num_tokens": 157222278.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.24800286157147966,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.0004516786065458632,
+      "loss": 1.311,
+      "mean_token_accuracy": 0.7028737253509462,
+      "num_tokens": 158989873.0,
+      "step": 910
+    },
+    {
+      "epoch": 0.2507281677425948,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.00045097626071077395,
+      "loss": 1.338,
+      "mean_token_accuracy": 0.700885118637234,
+      "num_tokens": 160740690.0,
+      "step": 920
+    },
+    {
+      "epoch": 0.25345347391371,
+      "grad_norm": 0.466796875,
+      "learning_rate": 0.0004502739148756848,
+      "loss": 1.3453,
+      "mean_token_accuracy": 0.6962541362270713,
+      "num_tokens": 162505210.0,
+      "step": 930
+    },
+    {
+      "epoch": 0.25617878008482514,
+      "grad_norm": 0.474609375,
+      "learning_rate": 0.0004495715690405956,
+      "loss": 1.3439,
+      "mean_token_accuracy": 0.6976133125834167,
+      "num_tokens": 164184119.0,
+      "step": 940
+    },
+    {
+      "epoch": 0.2589040862559403,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.0004488692232055064,
+      "loss": 1.296,
+      "mean_token_accuracy": 0.7053397912532091,
+      "num_tokens": 165971146.0,
+      "step": 950
+    },
+    {
+      "epoch": 0.26162939242705546,
+      "grad_norm": 0.45703125,
+      "learning_rate": 0.0004481668773704172,
+      "loss": 1.3388,
+      "mean_token_accuracy": 0.7005649470724166,
+      "num_tokens": 167701701.0,
+      "step": 960
+    },
+    {
+      "epoch": 0.2643546985981706,
+      "grad_norm": 0.53125,
+      "learning_rate": 0.000447464531535328,
+      "loss": 1.3452,
+      "mean_token_accuracy": 0.6994968581944704,
+      "num_tokens": 169443326.0,
+      "step": 970
+    },
+    {
+      "epoch": 0.2670800047692858,
+      "grad_norm": 0.431640625,
+      "learning_rate": 0.0004467621857002388,
+      "loss": 1.3342,
+      "mean_token_accuracy": 0.6998818466439843,
+      "num_tokens": 171199385.0,
+      "step": 980
+    },
+    {
+      "epoch": 0.26980531094040094,
+      "grad_norm": 0.45703125,
+      "learning_rate": 0.0004460598398651496,
+      "loss": 1.2848,
+      "mean_token_accuracy": 0.7084928262978792,
+      "num_tokens": 172929493.0,
+      "step": 990
+    },
+    {
+      "epoch": 0.2725306171115161,
+      "grad_norm": 0.67578125,
+      "learning_rate": 0.0004453574940300604,
+      "loss": 1.2857,
+      "mean_token_accuracy": 0.7073307786136865,
+      "num_tokens": 174672601.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.27525592328263127,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.00044465514819497123,
+      "loss": 1.3244,
+      "mean_token_accuracy": 0.7018027381040156,
+      "num_tokens": 176441962.0,
+      "step": 1010
+    },
+    {
+      "epoch": 0.2779812294537464,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.000443952802359882,
+      "loss": 1.2832,
+      "mean_token_accuracy": 0.7082092766650021,
+      "num_tokens": 178197361.0,
+      "step": 1020
+    },
+    {
+      "epoch": 0.2807065356248616,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.00044325045652479286,
+      "loss": 1.3295,
+      "mean_token_accuracy": 0.6989543081261218,
+      "num_tokens": 179887580.0,
+      "step": 1030
+    },
+    {
+      "epoch": 0.28343184179597675,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0004425481106897036,
+      "loss": 1.2738,
+      "mean_token_accuracy": 0.7097910531796515,
+      "num_tokens": 181655865.0,
+      "step": 1040
+    },
+    {
+      "epoch": 0.2861571479670919,
+      "grad_norm": 0.4921875,
+      "learning_rate": 0.00044184576485461444,
+      "loss": 1.3083,
+      "mean_token_accuracy": 0.7053726987913251,
+      "num_tokens": 183445880.0,
+      "step": 1050
+    },
+    {
+      "epoch": 0.28888245413820707,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0004411434190195252,
+      "loss": 1.2624,
+      "mean_token_accuracy": 0.7115490132942796,
+      "num_tokens": 185175706.0,
+      "step": 1060
+    },
+    {
+      "epoch": 0.29160776030932223,
+      "grad_norm": 0.498046875,
+      "learning_rate": 0.00044044107318443607,
+      "loss": 1.2847,
+      "mean_token_accuracy": 0.7070434625260532,
+      "num_tokens": 186873396.0,
+      "step": 1070
+    },
+    {
+      "epoch": 0.2943330664804374,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.00043973872734934683,
+      "loss": 1.3302,
+      "mean_token_accuracy": 0.6996699880808592,
+      "num_tokens": 188591288.0,
+      "step": 1080
+    },
+    {
+      "epoch": 0.29705837265155255,
+      "grad_norm": 0.4375,
+      "learning_rate": 0.00043903638151425765,
+      "loss": 1.318,
+      "mean_token_accuracy": 0.7020871014334261,
+      "num_tokens": 190375182.0,
+      "step": 1090
+    },
+    {
+      "epoch": 0.2997836788226677,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.0004383340356791684,
+      "loss": 1.2928,
+      "mean_token_accuracy": 0.7056466254405678,
+      "num_tokens": 192104609.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.30250898499378287,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0004376316898440792,
+      "loss": 1.202,
+      "mean_token_accuracy": 0.7222756736911833,
+      "num_tokens": 193819768.0,
+      "step": 1110
+    },
+    {
+      "epoch": 0.30523429116489803,
+      "grad_norm": 0.439453125,
+      "learning_rate": 0.00043692934400899004,
+      "loss": 1.2824,
+      "mean_token_accuracy": 0.7082328143529594,
+      "num_tokens": 195541705.0,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3079595973360132,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.0004362269981739008,
+      "loss": 1.2729,
+      "mean_token_accuracy": 0.7094620336778462,
+      "num_tokens": 197250976.0,
+      "step": 1130
+    },
+    {
+      "epoch": 0.31068490350712835,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.00043552465233881167,
+      "loss": 1.2343,
+      "mean_token_accuracy": 0.7161834868602455,
+      "num_tokens": 198964087.0,
+      "step": 1140
+    },
+    {
+      "epoch": 0.31341020967824357,
+      "grad_norm": 0.408203125,
+      "learning_rate": 0.00043482230650372243,
+      "loss": 1.2853,
+      "mean_token_accuracy": 0.7067827261053026,
+      "num_tokens": 200654341.0,
+      "step": 1150
+    },
+    {
+      "epoch": 0.31613551584935873,
+      "grad_norm": 0.470703125,
+      "learning_rate": 0.00043411996066863325,
+      "loss": 1.3217,
+      "mean_token_accuracy": 0.7011503103189171,
+      "num_tokens": 202410389.0,
+      "step": 1160
+    },
+    {
+      "epoch": 0.3188608220204739,
+      "grad_norm": 0.43359375,
+      "learning_rate": 0.000433417614833544,
+      "loss": 1.2703,
+      "mean_token_accuracy": 0.710265371389687,
+      "num_tokens": 204176073.0,
+      "step": 1170
+    },
+    {
+      "epoch": 0.32158612819158905,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0004327152689984549,
+      "loss": 1.257,
+      "mean_token_accuracy": 0.713060013204813,
+      "num_tokens": 206017214.0,
+      "step": 1180
+    },
+    {
+      "epoch": 0.3243114343627042,
+      "grad_norm": 0.453125,
+      "learning_rate": 0.00043201292316336564,
+      "loss": 1.2758,
+      "mean_token_accuracy": 0.7083034078590572,
+      "num_tokens": 207832984.0,
+      "step": 1190
+    },
+    {
+      "epoch": 0.32703674053381937,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.00043131057732827646,
+      "loss": 1.2757,
+      "mean_token_accuracy": 0.7085663201287389,
+      "num_tokens": 209533427.0,
+      "step": 1200
+    },
+    {
+      "epoch": 0.32976204670493453,
+      "grad_norm": 0.443359375,
+      "learning_rate": 0.0004306082314931872,
+      "loss": 1.2609,
+      "mean_token_accuracy": 0.7120108696632087,
+      "num_tokens": 211344332.0,
+      "step": 1210
+    },
+    {
+      "epoch": 0.3324873528760497,
+      "grad_norm": 0.474609375,
+      "learning_rate": 0.0004299058856580981,
+      "loss": 1.2847,
+      "mean_token_accuracy": 0.7070382171310484,
+      "num_tokens": 213088746.0,
+      "step": 1220
+    },
+    {
+      "epoch": 0.33521265904716485,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.00042920353982300885,
+      "loss": 1.2914,
+      "mean_token_accuracy": 0.7046342886984348,
+      "num_tokens": 214889528.0,
+      "step": 1230
+    },
+    {
+      "epoch": 0.33793796521828,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.00042850119398791966,
+      "loss": 1.318,
+      "mean_token_accuracy": 0.7027574786916375,
+      "num_tokens": 216603081.0,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3406632713893952,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0004277988481528305,
+      "loss": 1.2342,
+      "mean_token_accuracy": 0.7151992056518793,
+      "num_tokens": 218292845.0,
+      "step": 1250
+    },
+    {
+      "epoch": 0.34338857756051033,
+      "grad_norm": 0.466796875,
+      "learning_rate": 0.0004270965023177413,
+      "loss": 1.273,
+      "mean_token_accuracy": 0.7111638378351927,
+      "num_tokens": 220029696.0,
+      "step": 1260
+    },
+    {
+      "epoch": 0.3461138837316255,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.00042639415648265206,
+      "loss": 1.3345,
+      "mean_token_accuracy": 0.699883119482547,
+      "num_tokens": 221810553.0,
+      "step": 1270
+    },
+    {
+      "epoch": 0.34883918990274065,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.00042569181064756287,
+      "loss": 1.2527,
+      "mean_token_accuracy": 0.7150985508225858,
+      "num_tokens": 223552801.0,
+      "step": 1280
+    },
+    {
+      "epoch": 0.3515644960738558,
+      "grad_norm": 0.43359375,
+      "learning_rate": 0.0004249894648124737,
+      "loss": 1.2599,
+      "mean_token_accuracy": 0.7120441010221839,
+      "num_tokens": 225254929.0,
+      "step": 1290
+    },
+    {
+      "epoch": 0.354289802244971,
+      "grad_norm": 0.49609375,
+      "learning_rate": 0.0004242871189773845,
+      "loss": 1.2961,
+      "mean_token_accuracy": 0.7060732362791896,
+      "num_tokens": 227017304.0,
+      "step": 1300
+    },
+    {
+      "epoch": 0.35701510841608614,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.00042358477314229526,
+      "loss": 1.2191,
+      "mean_token_accuracy": 0.7194700758904219,
+      "num_tokens": 228785771.0,
+      "step": 1310
+    },
+    {
+      "epoch": 0.3597404145872013,
+      "grad_norm": 0.3828125,
+      "learning_rate": 0.0004228824273072061,
+      "loss": 1.2231,
+      "mean_token_accuracy": 0.7179213220253586,
+      "num_tokens": 230489667.0,
+      "step": 1320
+    },
+    {
+      "epoch": 0.36246572075831646,
+      "grad_norm": 0.408203125,
+      "learning_rate": 0.0004221800814721169,
+      "loss": 1.2519,
+      "mean_token_accuracy": 0.7132132126018405,
+      "num_tokens": 232269581.0,
+      "step": 1330
+    },
+    {
+      "epoch": 0.3651910269294316,
+      "grad_norm": 0.42578125,
+      "learning_rate": 0.00042147773563702766,
+      "loss": 1.2605,
+      "mean_token_accuracy": 0.7092904560267925,
+      "num_tokens": 234028126.0,
+      "step": 1340
+    },
+    {
+      "epoch": 0.3679163331005468,
+      "grad_norm": 0.53515625,
+      "learning_rate": 0.00042077538980193847,
+      "loss": 1.2882,
+      "mean_token_accuracy": 0.7081171317026019,
+      "num_tokens": 235817129.0,
+      "step": 1350
+    },
+    {
+      "epoch": 0.37064163927166194,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0004200730439668493,
+      "loss": 1.2574,
+      "mean_token_accuracy": 0.7129102074541152,
+      "num_tokens": 237588984.0,
+      "step": 1360
+    },
+    {
+      "epoch": 0.3733669454427771,
+      "grad_norm": 0.44921875,
+      "learning_rate": 0.0004193706981317601,
+      "loss": 1.2072,
+      "mean_token_accuracy": 0.7199260313995183,
+      "num_tokens": 239305200.0,
+      "step": 1370
+    },
+    {
+      "epoch": 0.37609225161389226,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.00041866835229667086,
+      "loss": 1.2353,
+      "mean_token_accuracy": 0.7163059611804783,
+      "num_tokens": 241078759.0,
+      "step": 1380
+    },
+    {
+      "epoch": 0.3788175577850074,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.0004179660064615817,
+      "loss": 1.2712,
+      "mean_token_accuracy": 0.7112471625208855,
+      "num_tokens": 242876841.0,
+      "step": 1390
+    },
+    {
+      "epoch": 0.3815428639561226,
+      "grad_norm": 0.40234375,
+      "learning_rate": 0.0004172636606264925,
+      "loss": 1.2176,
+      "mean_token_accuracy": 0.7165075090713799,
+      "num_tokens": 244578724.0,
+      "step": 1400
+    },
+    {
+      "epoch": 0.38426817012723774,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0004165613147914033,
+      "loss": 1.2526,
+      "mean_token_accuracy": 0.7108958872035146,
+      "num_tokens": 246363009.0,
+      "step": 1410
+    },
+    {
+      "epoch": 0.3869934762983529,
+      "grad_norm": 0.431640625,
+      "learning_rate": 0.00041585896895631407,
+      "loss": 1.293,
+      "mean_token_accuracy": 0.7064874111674726,
+      "num_tokens": 248120613.0,
+      "step": 1420
+    },
+    {
+      "epoch": 0.38971878246946806,
+      "grad_norm": 0.42578125,
+      "learning_rate": 0.0004151566231212249,
+      "loss": 1.2606,
+      "mean_token_accuracy": 0.7118613218888641,
+      "num_tokens": 249941867.0,
+      "step": 1430
+    },
+    {
+      "epoch": 0.3924440886405832,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0004144542772861357,
+      "loss": 1.2136,
+      "mean_token_accuracy": 0.7203867037780582,
+      "num_tokens": 251695734.0,
+      "step": 1440
+    },
+    {
+      "epoch": 0.3951693948116984,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0004137519314510465,
+      "loss": 1.2663,
+      "mean_token_accuracy": 0.7106507356278599,
+      "num_tokens": 253503211.0,
+      "step": 1450
+    },
+    {
+      "epoch": 0.39789470098281354,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0004130495856159573,
+      "loss": 1.2482,
+      "mean_token_accuracy": 0.7142273456789553,
+      "num_tokens": 255313734.0,
+      "step": 1460
+    },
+    {
+      "epoch": 0.4006200071539287,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.00041234723978086815,
+      "loss": 1.1809,
+      "mean_token_accuracy": 0.7257948698475957,
+      "num_tokens": 257016513.0,
+      "step": 1470
+    },
+    {
+      "epoch": 0.40334531332504386,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.0004116448939457789,
+      "loss": 1.2676,
+      "mean_token_accuracy": 0.7092947035096586,
+      "num_tokens": 258760362.0,
+      "step": 1480
+    },
+    {
+      "epoch": 0.406070619496159,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.0004109425481106897,
+      "loss": 1.2095,
+      "mean_token_accuracy": 0.7201486421748996,
+      "num_tokens": 260546971.0,
+      "step": 1490
+    },
+    {
+      "epoch": 0.4087959256672742,
+      "grad_norm": 0.431640625,
+      "learning_rate": 0.0004102402022756005,
+      "loss": 1.2297,
+      "mean_token_accuracy": 0.7155070326291024,
+      "num_tokens": 262307672.0,
+      "step": 1500
+    },
+    {
+      "epoch": 0.41152123183838935,
+      "grad_norm": 0.427734375,
+      "learning_rate": 0.00040953785644051136,
+      "loss": 1.2367,
+      "mean_token_accuracy": 0.7149525980465115,
+      "num_tokens": 264023889.0,
+      "step": 1510
+    },
+    {
+      "epoch": 0.4142465380095045,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.0004088355106054221,
+      "loss": 1.2325,
+      "mean_token_accuracy": 0.7151728126220405,
+      "num_tokens": 265798519.0,
+      "step": 1520
+    },
+    {
+      "epoch": 0.41697184418061967,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.00040813316477033293,
+      "loss": 1.2632,
+      "mean_token_accuracy": 0.710959506034851,
+      "num_tokens": 267546100.0,
+      "step": 1530
+    },
+    {
+      "epoch": 0.4196971503517348,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0004074308189352437,
+      "loss": 1.2144,
+      "mean_token_accuracy": 0.7181764165870845,
+      "num_tokens": 269297641.0,
+      "step": 1540
+    },
+    {
+      "epoch": 0.42242245652285,
+      "grad_norm": 0.3984375,
+      "learning_rate": 0.00040672847310015456,
+      "loss": 1.261,
+      "mean_token_accuracy": 0.710538933891803,
+      "num_tokens": 271053006.0,
+      "step": 1550
+    },
+    {
+      "epoch": 0.42514776269396515,
+      "grad_norm": 0.44140625,
+      "learning_rate": 0.0004060261272650653,
+      "loss": 1.2391,
+      "mean_token_accuracy": 0.7155305192805826,
+      "num_tokens": 272829487.0,
+      "step": 1560
+    },
+    {
+      "epoch": 0.4278730688650803,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0004053237814299761,
+      "loss": 1.2266,
+      "mean_token_accuracy": 0.7166890133172273,
+      "num_tokens": 274621990.0,
+      "step": 1570
+    },
+    {
+      "epoch": 0.43059837503619547,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.00040462143559488696,
+      "loss": 1.2463,
+      "mean_token_accuracy": 0.7121757534332573,
+      "num_tokens": 276398989.0,
+      "step": 1580
+    },
+    {
+      "epoch": 0.43332368120731063,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0004039190897597977,
+      "loss": 1.2152,
+      "mean_token_accuracy": 0.7180015265010298,
+      "num_tokens": 278192884.0,
+      "step": 1590
+    },
+    {
+      "epoch": 0.4360489873784258,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.00040321674392470853,
+      "loss": 1.204,
+      "mean_token_accuracy": 0.7211478114128113,
+      "num_tokens": 279937931.0,
+      "step": 1600
+    },
+    {
+      "epoch": 0.43877429354954095,
+      "grad_norm": 0.375,
+      "learning_rate": 0.0004025143980896193,
+      "loss": 1.2067,
+      "mean_token_accuracy": 0.7212561624124646,
+      "num_tokens": 281698143.0,
+      "step": 1610
+    },
+    {
+      "epoch": 0.4414995997206561,
+      "grad_norm": 0.412109375,
+      "learning_rate": 0.00040181205225453017,
+      "loss": 1.2264,
+      "mean_token_accuracy": 0.7175595877692104,
+      "num_tokens": 283412565.0,
+      "step": 1620
+    },
+    {
+      "epoch": 0.44422490589177127,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.0004011097064194409,
+      "loss": 1.2273,
+      "mean_token_accuracy": 0.7161778485402465,
+      "num_tokens": 285184244.0,
+      "step": 1630
+    },
+    {
+      "epoch": 0.44695021206288643,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.00040040736058435174,
+      "loss": 1.1504,
+      "mean_token_accuracy": 0.7301577464677393,
+      "num_tokens": 286963226.0,
+      "step": 1640
+    },
+    {
+      "epoch": 0.4496755182340016,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.0003997050147492625,
+      "loss": 1.1838,
+      "mean_token_accuracy": 0.7249205836094916,
+      "num_tokens": 288742524.0,
+      "step": 1650
+    },
+    {
+      "epoch": 0.45240082440511675,
+      "grad_norm": 0.44921875,
+      "learning_rate": 0.00039900266891417337,
+      "loss": 1.2179,
+      "mean_token_accuracy": 0.71893819719553,
+      "num_tokens": 290486439.0,
+      "step": 1660
+    },
+    {
+      "epoch": 0.4551261305762319,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.00039830032307908413,
+      "loss": 1.2206,
+      "mean_token_accuracy": 0.7182010743767023,
+      "num_tokens": 292244616.0,
+      "step": 1670
+    },
+    {
+      "epoch": 0.4578514367473471,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.00039759797724399495,
+      "loss": 1.2252,
+      "mean_token_accuracy": 0.7172848870046437,
+      "num_tokens": 294028863.0,
+      "step": 1680
+    },
+    {
+      "epoch": 0.46057674291846223,
+      "grad_norm": 0.408203125,
+      "learning_rate": 0.00039689563140890577,
+      "loss": 1.1818,
+      "mean_token_accuracy": 0.723115173354745,
+      "num_tokens": 295842660.0,
+      "step": 1690
+    },
+    {
+      "epoch": 0.4633020490895774,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0003961932855738166,
+      "loss": 1.1913,
+      "mean_token_accuracy": 0.7237382984720171,
+      "num_tokens": 297606076.0,
+      "step": 1700
+    },
+    {
+      "epoch": 0.46602735526069256,
+      "grad_norm": 0.490234375,
+      "learning_rate": 0.00039549093973872734,
+      "loss": 1.2381,
+      "mean_token_accuracy": 0.714545346237719,
+      "num_tokens": 299367632.0,
+      "step": 1710
+    },
+    {
+      "epoch": 0.4687526614318077,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.00039478859390363816,
+      "loss": 1.1702,
+      "mean_token_accuracy": 0.7281134461984038,
+      "num_tokens": 301097990.0,
+      "step": 1720
+    },
+    {
+      "epoch": 0.4714779676029229,
+      "grad_norm": 0.431640625,
+      "learning_rate": 0.000394086248068549,
+      "loss": 1.1872,
+      "mean_token_accuracy": 0.7246512384153903,
+      "num_tokens": 302847308.0,
+      "step": 1730
+    },
+    {
+      "epoch": 0.47420327377403804,
+      "grad_norm": 0.421875,
+      "learning_rate": 0.0003933839022334598,
+      "loss": 1.2085,
+      "mean_token_accuracy": 0.7196571965701878,
+      "num_tokens": 304582625.0,
+      "step": 1740
+    },
+    {
+      "epoch": 0.4769285799451532,
+      "grad_norm": 0.44140625,
+      "learning_rate": 0.00039268155639837055,
+      "loss": 1.1827,
+      "mean_token_accuracy": 0.7259564273059368,
+      "num_tokens": 306358206.0,
+      "step": 1750
+    },
+    {
+      "epoch": 0.47965388611626836,
+      "grad_norm": 0.40234375,
+      "learning_rate": 0.00039197921056328137,
+      "loss": 1.1772,
+      "mean_token_accuracy": 0.7265206996351481,
+      "num_tokens": 308026562.0,
+      "step": 1760
+    },
+    {
+      "epoch": 0.4823791922873835,
+      "grad_norm": 0.57421875,
+      "learning_rate": 0.0003912768647281922,
+      "loss": 1.2166,
+      "mean_token_accuracy": 0.7178250108845532,
+      "num_tokens": 309804543.0,
+      "step": 1770
+    },
+    {
+      "epoch": 0.4851044984584987,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.000390574518893103,
+      "loss": 1.214,
+      "mean_token_accuracy": 0.7180218786001206,
+      "num_tokens": 311549771.0,
+      "step": 1780
+    },
+    {
+      "epoch": 0.48782980462961384,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.00038987217305801376,
+      "loss": 1.2368,
+      "mean_token_accuracy": 0.7157372819259763,
+      "num_tokens": 313326889.0,
+      "step": 1790
+    },
+    {
+      "epoch": 0.490555110800729,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0003891698272229246,
+      "loss": 1.1708,
+      "mean_token_accuracy": 0.7274354211986065,
+      "num_tokens": 315064715.0,
+      "step": 1800
+    },
+    {
+      "epoch": 0.49328041697184416,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.0003884674813878354,
+      "loss": 1.1907,
+      "mean_token_accuracy": 0.7222747911699117,
+      "num_tokens": 316809509.0,
+      "step": 1810
+    },
+    {
+      "epoch": 0.4960057231429593,
+      "grad_norm": 0.40234375,
+      "learning_rate": 0.00038776513555274615,
+      "loss": 1.1724,
+      "mean_token_accuracy": 0.7267968454398215,
+      "num_tokens": 318597481.0,
+      "step": 1820
+    },
+    {
+      "epoch": 0.4987310293140745,
+      "grad_norm": 0.796875,
+      "learning_rate": 0.00038706278971765697,
+      "loss": 1.1559,
+      "mean_token_accuracy": 0.7283987457863986,
+      "num_tokens": 320382863.0,
+      "step": 1830
+    },
+    {
+      "epoch": 0.5014563354851896,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003863604438825678,
+      "loss": 1.1879,
+      "mean_token_accuracy": 0.724396638572216,
+      "num_tokens": 322136431.0,
+      "step": 1840
+    },
+    {
+      "epoch": 0.5041816416563049,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.0003856580980474786,
+      "loss": 1.1686,
+      "mean_token_accuracy": 0.7275835652835667,
+      "num_tokens": 323892089.0,
+      "step": 1850
+    },
+    {
+      "epoch": 0.50690694782742,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.00038495575221238936,
+      "loss": 1.1808,
+      "mean_token_accuracy": 0.7243333254009485,
+      "num_tokens": 325693457.0,
+      "step": 1860
+    },
+    {
+      "epoch": 0.5096322539985352,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.0003842534063773002,
+      "loss": 1.1474,
+      "mean_token_accuracy": 0.7292418093420565,
+      "num_tokens": 327397884.0,
+      "step": 1870
+    },
+    {
+      "epoch": 0.5123575601696503,
+      "grad_norm": 0.4140625,
+      "learning_rate": 0.000383551060542211,
+      "loss": 1.2087,
+      "mean_token_accuracy": 0.7199928862042725,
+      "num_tokens": 329077502.0,
+      "step": 1880
+    },
+    {
+      "epoch": 0.5150828663407655,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0003828487147071218,
+      "loss": 1.1952,
+      "mean_token_accuracy": 0.7225228334777057,
+      "num_tokens": 330847695.0,
+      "step": 1890
+    },
+    {
+      "epoch": 0.5178081725118806,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.00038214636887203257,
+      "loss": 1.1591,
+      "mean_token_accuracy": 0.7250943325459958,
+      "num_tokens": 332578366.0,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5205334786829958,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.00038144402303694344,
+      "loss": 1.1994,
+      "mean_token_accuracy": 0.7206506300717592,
+      "num_tokens": 334293589.0,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5232587848541109,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003807416772018542,
+      "loss": 1.1841,
+      "mean_token_accuracy": 0.7210906410589815,
+      "num_tokens": 336016972.0,
+      "step": 1920
+    },
+    {
+      "epoch": 0.5259840910252261,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.000380039331366765,
+      "loss": 1.1503,
+      "mean_token_accuracy": 0.7291983521543444,
+      "num_tokens": 337805977.0,
+      "step": 1930
+    },
+    {
+      "epoch": 0.5287093971963412,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0003793369855316758,
+      "loss": 1.1218,
+      "mean_token_accuracy": 0.7343028046190738,
+      "num_tokens": 339548221.0,
+      "step": 1940
+    },
+    {
+      "epoch": 0.5314347033674565,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.00037863463969658664,
+      "loss": 1.2138,
+      "mean_token_accuracy": 0.7172385750338435,
+      "num_tokens": 341268344.0,
+      "step": 1950
+    },
+    {
+      "epoch": 0.5341600095385716,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0003779322938614974,
+      "loss": 1.1792,
+      "mean_token_accuracy": 0.724903620686382,
+      "num_tokens": 343017522.0,
+      "step": 1960
+    },
+    {
+      "epoch": 0.5368853157096868,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.0003772299480264082,
+      "loss": 1.1665,
+      "mean_token_accuracy": 0.7268314240500331,
+      "num_tokens": 344699192.0,
+      "step": 1970
+    },
+    {
+      "epoch": 0.5396106218808019,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.00037652760219131904,
+      "loss": 1.143,
+      "mean_token_accuracy": 0.7308998920954763,
+      "num_tokens": 346458527.0,
+      "step": 1980
+    },
+    {
+      "epoch": 0.5423359280519171,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.00037582525635622985,
+      "loss": 1.1795,
+      "mean_token_accuracy": 0.7253339084796607,
+      "num_tokens": 348204585.0,
+      "step": 1990
+    },
+    {
+      "epoch": 0.5450612342230322,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0003751229105211406,
+      "loss": 1.1936,
+      "mean_token_accuracy": 0.7219026086851954,
+      "num_tokens": 349928259.0,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5477865403941474,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.00037442056468605143,
+      "loss": 1.1629,
+      "mean_token_accuracy": 0.7267959822900594,
+      "num_tokens": 351568018.0,
+      "step": 2010
+    },
+    {
+      "epoch": 0.5505118465652625,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.00037371821885096224,
+      "loss": 1.1492,
+      "mean_token_accuracy": 0.7309575733728707,
+      "num_tokens": 353325716.0,
+      "step": 2020
+    },
+    {
+      "epoch": 0.5532371527363777,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.000373015873015873,
+      "loss": 1.1898,
+      "mean_token_accuracy": 0.7220692792907357,
+      "num_tokens": 355099655.0,
+      "step": 2030
+    },
+    {
+      "epoch": 0.5559624589074929,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0003723135271807838,
+      "loss": 1.159,
+      "mean_token_accuracy": 0.7283659107983113,
+      "num_tokens": 356892448.0,
+      "step": 2040
+    },
+    {
+      "epoch": 0.5586877650786081,
+      "grad_norm": 0.41015625,
+      "learning_rate": 0.0003716111813456946,
+      "loss": 1.1855,
+      "mean_token_accuracy": 0.7233272660523653,
+      "num_tokens": 358599855.0,
+      "step": 2050
+    },
+    {
+      "epoch": 0.5614130712497232,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.00037090883551060545,
+      "loss": 1.2008,
+      "mean_token_accuracy": 0.7202940073795616,
+      "num_tokens": 360366528.0,
+      "step": 2060
+    },
+    {
+      "epoch": 0.5641383774208384,
+      "grad_norm": 0.396484375,
+      "learning_rate": 0.0003702064896755162,
+      "loss": 1.1701,
+      "mean_token_accuracy": 0.7270756110548973,
+      "num_tokens": 362123903.0,
+      "step": 2070
+    },
+    {
+      "epoch": 0.5668636835919535,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.00036950414384042703,
+      "loss": 1.145,
+      "mean_token_accuracy": 0.732559056673199,
+      "num_tokens": 363907270.0,
+      "step": 2080
+    },
+    {
+      "epoch": 0.5695889897630687,
+      "grad_norm": 0.416015625,
+      "learning_rate": 0.00036880179800533784,
+      "loss": 1.1266,
+      "mean_token_accuracy": 0.7330201560631394,
+      "num_tokens": 365602883.0,
+      "step": 2090
+    },
+    {
+      "epoch": 0.5723142959341838,
+      "grad_norm": 0.419921875,
+      "learning_rate": 0.00036809945217024866,
+      "loss": 1.1442,
+      "mean_token_accuracy": 0.7297326970845461,
+      "num_tokens": 367308921.0,
+      "step": 2100
+    },
+    {
+      "epoch": 0.575039602105299,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0003673971063351594,
+      "loss": 1.1582,
+      "mean_token_accuracy": 0.7281600207090377,
+      "num_tokens": 369068618.0,
+      "step": 2110
+    },
+    {
+      "epoch": 0.5777649082764141,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.00036669476050007024,
+      "loss": 1.1683,
+      "mean_token_accuracy": 0.727477038372308,
+      "num_tokens": 370792272.0,
+      "step": 2120
+    },
+    {
+      "epoch": 0.5804902144475294,
+      "grad_norm": 0.375,
+      "learning_rate": 0.00036599241466498105,
+      "loss": 1.1724,
+      "mean_token_accuracy": 0.725747492350638,
+      "num_tokens": 372506541.0,
+      "step": 2130
+    },
+    {
+      "epoch": 0.5832155206186445,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.00036529006882989187,
+      "loss": 1.1075,
+      "mean_token_accuracy": 0.7377508584409952,
+      "num_tokens": 374308542.0,
+      "step": 2140
+    },
+    {
+      "epoch": 0.5859408267897597,
+      "grad_norm": 0.47265625,
+      "learning_rate": 0.00036458772299480263,
+      "loss": 1.1207,
+      "mean_token_accuracy": 0.735583251528442,
+      "num_tokens": 376016266.0,
+      "step": 2150
+    },
+    {
+      "epoch": 0.5886661329608748,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.00036388537715971344,
+      "loss": 1.1742,
+      "mean_token_accuracy": 0.7264269077219069,
+      "num_tokens": 377732713.0,
+      "step": 2160
+    },
+    {
+      "epoch": 0.59139143913199,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.00036318303132462426,
+      "loss": 1.1541,
+      "mean_token_accuracy": 0.7288526111282408,
+      "num_tokens": 379457927.0,
+      "step": 2170
+    },
+    {
+      "epoch": 0.5941167453031051,
+      "grad_norm": 0.326171875,
+      "learning_rate": 0.0003624806854895351,
+      "loss": 1.2054,
+      "mean_token_accuracy": 0.719714920129627,
+      "num_tokens": 381311531.0,
+      "step": 2180
+    },
+    {
+      "epoch": 0.5968420514742203,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.00036177833965444584,
+      "loss": 1.1941,
+      "mean_token_accuracy": 0.7239844122901559,
+      "num_tokens": 383076036.0,
+      "step": 2190
+    },
+    {
+      "epoch": 0.5995673576453354,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0003610759938193567,
+      "loss": 1.181,
+      "mean_token_accuracy": 0.7247908087447286,
+      "num_tokens": 384869981.0,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6022926638164506,
+      "grad_norm": 0.408203125,
+      "learning_rate": 0.00036037364798426747,
+      "loss": 1.1548,
+      "mean_token_accuracy": 0.727875262312591,
+      "num_tokens": 386654361.0,
+      "step": 2210
+    },
+    {
+      "epoch": 0.6050179699875657,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.0003596713021491783,
+      "loss": 1.1284,
+      "mean_token_accuracy": 0.7336899240501225,
+      "num_tokens": 388405388.0,
+      "step": 2220
+    },
+    {
+      "epoch": 0.607743276158681,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.00035896895631408904,
+      "loss": 1.1629,
+      "mean_token_accuracy": 0.72736093737185,
+      "num_tokens": 390160411.0,
+      "step": 2230
+    },
+    {
+      "epoch": 0.6104685823297961,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.0003582666104789999,
+      "loss": 1.1718,
+      "mean_token_accuracy": 0.724588902015239,
+      "num_tokens": 391963393.0,
+      "step": 2240
+    },
+    {
+      "epoch": 0.6131938885009113,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0003575642646439107,
+      "loss": 1.1686,
+      "mean_token_accuracy": 0.7265552974306047,
+      "num_tokens": 393719494.0,
+      "step": 2250
+    },
+    {
+      "epoch": 0.6159191946720264,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.00035686191880882144,
+      "loss": 1.1597,
+      "mean_token_accuracy": 0.7278934679925442,
+      "num_tokens": 395419867.0,
+      "step": 2260
+    },
+    {
+      "epoch": 0.6186445008431416,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.00035615957297373225,
+      "loss": 1.14,
+      "mean_token_accuracy": 0.7317047557793558,
+      "num_tokens": 397164636.0,
+      "step": 2270
+    },
+    {
+      "epoch": 0.6213698070142567,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.00035545722713864307,
+      "loss": 1.1711,
+      "mean_token_accuracy": 0.7251532531343401,
+      "num_tokens": 398897880.0,
+      "step": 2280
+    },
+    {
+      "epoch": 0.6240951131853719,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003547548813035539,
+      "loss": 1.1499,
+      "mean_token_accuracy": 0.72917456375435,
+      "num_tokens": 400668616.0,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6268204193564871,
+      "grad_norm": 0.375,
+      "learning_rate": 0.00035405253546846464,
+      "loss": 1.1378,
+      "mean_token_accuracy": 0.7313735852017998,
+      "num_tokens": 402379710.0,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6295457255276022,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0003533501896333755,
+      "loss": 1.1381,
+      "mean_token_accuracy": 0.7306974153965712,
+      "num_tokens": 404084345.0,
+      "step": 2310
+    },
+    {
+      "epoch": 0.6322710316987175,
+      "grad_norm": 0.3828125,
+      "learning_rate": 0.0003526478437982863,
+      "loss": 1.0999,
+      "mean_token_accuracy": 0.7397502162493765,
+      "num_tokens": 405770321.0,
+      "step": 2320
+    },
+    {
+      "epoch": 0.6349963378698326,
+      "grad_norm": 0.3359375,
+      "learning_rate": 0.0003519454979631971,
+      "loss": 1.1241,
+      "mean_token_accuracy": 0.733117110747844,
+      "num_tokens": 407492020.0,
+      "step": 2330
+    },
+    {
+      "epoch": 0.6377216440409478,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.00035124315212810785,
+      "loss": 1.1205,
+      "mean_token_accuracy": 0.7354815387167036,
+      "num_tokens": 409257115.0,
+      "step": 2340
+    },
+    {
+      "epoch": 0.6404469502120629,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0003505408062930187,
+      "loss": 1.1524,
+      "mean_token_accuracy": 0.7284994964487851,
+      "num_tokens": 410986071.0,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6431722563831781,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0003498384604579295,
+      "loss": 1.1326,
+      "mean_token_accuracy": 0.73249177262187,
+      "num_tokens": 412763897.0,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6458975625542932,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003491361146228403,
+      "loss": 1.0881,
+      "mean_token_accuracy": 0.7415378532372415,
+      "num_tokens": 414503168.0,
+      "step": 2370
+    },
+    {
+      "epoch": 0.6486228687254084,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.00034843376878775106,
+      "loss": 1.1189,
+      "mean_token_accuracy": 0.7360304208472371,
+      "num_tokens": 416227145.0,
+      "step": 2380
+    },
+    {
+      "epoch": 0.6513481748965235,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00034773142295266193,
+      "loss": 1.1114,
+      "mean_token_accuracy": 0.7346363394521177,
+      "num_tokens": 417973040.0,
+      "step": 2390
+    },
+    {
+      "epoch": 0.6540734810676387,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0003470290771175727,
+      "loss": 1.1291,
+      "mean_token_accuracy": 0.7329517565667629,
+      "num_tokens": 419736780.0,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6567987872387538,
+      "grad_norm": 0.32421875,
+      "learning_rate": 0.0003463267312824835,
+      "loss": 1.12,
+      "mean_token_accuracy": 0.7355642972514034,
+      "num_tokens": 421516023.0,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6595240934098691,
+      "grad_norm": 0.3984375,
+      "learning_rate": 0.0003456243854473943,
+      "loss": 1.1714,
+      "mean_token_accuracy": 0.7273551743477583,
+      "num_tokens": 423324296.0,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6622493995809842,
+      "grad_norm": 0.42578125,
+      "learning_rate": 0.00034492203961230514,
+      "loss": 1.1472,
+      "mean_token_accuracy": 0.7267329922877253,
+      "num_tokens": 425114987.0,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6649747057520994,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0003442196937772159,
+      "loss": 1.114,
+      "mean_token_accuracy": 0.7362177189439535,
+      "num_tokens": 426875751.0,
+      "step": 2440
+    },
+    {
+      "epoch": 0.6677000119232145,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0003435173479421267,
+      "loss": 1.1187,
+      "mean_token_accuracy": 0.7348853600211441,
+      "num_tokens": 428715433.0,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6704253180943297,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.00034281500210703753,
+      "loss": 1.1587,
+      "mean_token_accuracy": 0.7272924329154193,
+      "num_tokens": 430443301.0,
+      "step": 2460
+    },
+    {
+      "epoch": 0.6731506242654448,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00034211265627194835,
+      "loss": 1.0693,
+      "mean_token_accuracy": 0.7440984978340566,
+      "num_tokens": 432149898.0,
+      "step": 2470
+    },
+    {
+      "epoch": 0.67587593043656,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0003414103104368591,
+      "loss": 1.0951,
+      "mean_token_accuracy": 0.7401705386117101,
+      "num_tokens": 433904775.0,
+      "step": 2480
+    },
+    {
+      "epoch": 0.6786012366076751,
+      "grad_norm": 0.3828125,
+      "learning_rate": 0.00034070796460176987,
+      "loss": 1.0992,
+      "mean_token_accuracy": 0.7375994463451206,
+      "num_tokens": 435648944.0,
+      "step": 2490
+    },
+    {
+      "epoch": 0.6813265427787903,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00034000561876668074,
+      "loss": 1.1405,
+      "mean_token_accuracy": 0.7315420118160546,
+      "num_tokens": 437390122.0,
+      "step": 2500
+    },
+    {
+      "epoch": 0.6840518489499054,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0003393032729315915,
+      "loss": 1.1311,
+      "mean_token_accuracy": 0.733266216237098,
+      "num_tokens": 439126311.0,
+      "step": 2510
+    },
+    {
+      "epoch": 0.6867771551210207,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0003386009270965023,
+      "loss": 1.1102,
+      "mean_token_accuracy": 0.7364032724872231,
+      "num_tokens": 440890933.0,
+      "step": 2520
+    },
+    {
+      "epoch": 0.6895024612921358,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.00033789858126141313,
+      "loss": 1.101,
+      "mean_token_accuracy": 0.7384627360850573,
+      "num_tokens": 442611770.0,
+      "step": 2530
+    },
+    {
+      "epoch": 0.692227767463251,
+      "grad_norm": 0.369140625,
+      "learning_rate": 0.00033719623542632395,
+      "loss": 1.1059,
+      "mean_token_accuracy": 0.7377724641934037,
+      "num_tokens": 444358166.0,
+      "step": 2540
+    },
+    {
+      "epoch": 0.6949530736343661,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003364938895912347,
+      "loss": 1.145,
+      "mean_token_accuracy": 0.7298734590411187,
+      "num_tokens": 446078790.0,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6976783798054813,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0003357915437561455,
+      "loss": 1.1079,
+      "mean_token_accuracy": 0.7355253180488944,
+      "num_tokens": 447811186.0,
+      "step": 2560
+    },
+    {
+      "epoch": 0.7004036859765964,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.00033508919792105634,
+      "loss": 1.124,
+      "mean_token_accuracy": 0.7344003698788584,
+      "num_tokens": 449537962.0,
+      "step": 2570
+    },
+    {
+      "epoch": 0.7031289921477116,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00033438685208596715,
+      "loss": 1.0841,
+      "mean_token_accuracy": 0.7412578398361802,
+      "num_tokens": 451259469.0,
+      "step": 2580
+    },
+    {
+      "epoch": 0.7058542983188267,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0003336845062508779,
+      "loss": 1.1074,
+      "mean_token_accuracy": 0.7388199215754867,
+      "num_tokens": 453041146.0,
+      "step": 2590
+    },
+    {
+      "epoch": 0.708579604489942,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.00033298216041578873,
+      "loss": 1.132,
+      "mean_token_accuracy": 0.7321982179768384,
+      "num_tokens": 454753651.0,
+      "step": 2600
+    },
+    {
+      "epoch": 0.711304910661057,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.00033227981458069955,
+      "loss": 1.0938,
+      "mean_token_accuracy": 0.7390209911391139,
+      "num_tokens": 456537057.0,
+      "step": 2610
+    },
+    {
+      "epoch": 0.7140302168321723,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.00033157746874561036,
+      "loss": 1.1417,
+      "mean_token_accuracy": 0.7310832546092569,
+      "num_tokens": 458271296.0,
+      "step": 2620
+    },
+    {
+      "epoch": 0.7167555230032874,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.0003308751229105211,
+      "loss": 1.1118,
+      "mean_token_accuracy": 0.7353935448452831,
+      "num_tokens": 460014932.0,
+      "step": 2630
+    },
+    {
+      "epoch": 0.7194808291744026,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.000330172777075432,
+      "loss": 1.1228,
+      "mean_token_accuracy": 0.7343842899426818,
+      "num_tokens": 461818480.0,
+      "step": 2640
+    },
+    {
+      "epoch": 0.7222061353455177,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.00032947043124034275,
+      "loss": 1.1535,
+      "mean_token_accuracy": 0.7270912263542414,
+      "num_tokens": 463519993.0,
+      "step": 2650
+    },
+    {
+      "epoch": 0.7249314415166329,
+      "grad_norm": 0.349609375,
+      "learning_rate": 0.00032876808540525357,
+      "loss": 1.1122,
+      "mean_token_accuracy": 0.7352983796037733,
+      "num_tokens": 465333101.0,
+      "step": 2660
+    },
+    {
+      "epoch": 0.727656747687748,
+      "grad_norm": 0.349609375,
+      "learning_rate": 0.00032806573957016433,
+      "loss": 1.1081,
+      "mean_token_accuracy": 0.7361832971684634,
+      "num_tokens": 467047198.0,
+      "step": 2670
+    },
+    {
+      "epoch": 0.7303820538588632,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003273633937350752,
+      "loss": 1.1227,
+      "mean_token_accuracy": 0.7353186239488423,
+      "num_tokens": 468806711.0,
+      "step": 2680
+    },
+    {
+      "epoch": 0.7331073600299783,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.00032666104789998596,
+      "loss": 1.0706,
+      "mean_token_accuracy": 0.7438032808713615,
+      "num_tokens": 470600782.0,
+      "step": 2690
+    },
+    {
+      "epoch": 0.7358326662010936,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0003259587020648968,
+      "loss": 1.1209,
+      "mean_token_accuracy": 0.734395441506058,
+      "num_tokens": 472349330.0,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7385579723722087,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.00032525635622980754,
+      "loss": 1.0917,
+      "mean_token_accuracy": 0.7406582017429173,
+      "num_tokens": 474106983.0,
+      "step": 2710
+    },
+    {
+      "epoch": 0.7412832785433239,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.00032455401039471835,
+      "loss": 1.146,
+      "mean_token_accuracy": 0.7296730799600482,
+      "num_tokens": 475902407.0,
+      "step": 2720
+    },
+    {
+      "epoch": 0.744008584714439,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.00032385166455962917,
+      "loss": 1.0453,
+      "mean_token_accuracy": 0.7479070770554245,
+      "num_tokens": 477654644.0,
+      "step": 2730
+    },
+    {
+      "epoch": 0.7467338908855542,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.00032314931872453993,
+      "loss": 1.1249,
+      "mean_token_accuracy": 0.7325184845365584,
+      "num_tokens": 479409078.0,
+      "step": 2740
+    },
+    {
+      "epoch": 0.7494591970566693,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003224469728894508,
+      "loss": 1.1047,
+      "mean_token_accuracy": 0.7368000699207187,
+      "num_tokens": 481154972.0,
+      "step": 2750
+    },
+    {
+      "epoch": 0.7521845032277845,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.00032174462705436156,
+      "loss": 1.1314,
+      "mean_token_accuracy": 0.7322432189248502,
+      "num_tokens": 482957554.0,
+      "step": 2760
+    },
+    {
+      "epoch": 0.7549098093988996,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003210422812192724,
+      "loss": 1.106,
+      "mean_token_accuracy": 0.7369892225600779,
+      "num_tokens": 484730289.0,
+      "step": 2770
+    },
+    {
+      "epoch": 0.7576351155700148,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.00032033993538418314,
+      "loss": 1.1143,
+      "mean_token_accuracy": 0.7335695104673505,
+      "num_tokens": 486390658.0,
+      "step": 2780
+    },
+    {
+      "epoch": 0.7603604217411299,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.000319637589549094,
+      "loss": 1.0688,
+      "mean_token_accuracy": 0.7441553009673954,
+      "num_tokens": 488084357.0,
+      "step": 2790
+    },
+    {
+      "epoch": 0.7630857279122452,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.00031893524371400477,
+      "loss": 1.1611,
+      "mean_token_accuracy": 0.7271939906291663,
+      "num_tokens": 489867504.0,
+      "step": 2800
+    },
+    {
+      "epoch": 0.7658110340833603,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0003182328978789156,
+      "loss": 1.1254,
+      "mean_token_accuracy": 0.7324933409690857,
+      "num_tokens": 491614120.0,
+      "step": 2810
+    },
+    {
+      "epoch": 0.7685363402544755,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.00031753055204382635,
+      "loss": 1.1173,
+      "mean_token_accuracy": 0.7357215932570398,
+      "num_tokens": 493360108.0,
+      "step": 2820
+    },
+    {
+      "epoch": 0.7712616464255906,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0003168282062087372,
+      "loss": 1.0859,
+      "mean_token_accuracy": 0.742600809969008,
+      "num_tokens": 495060867.0,
+      "step": 2830
+    },
+    {
+      "epoch": 0.7739869525967058,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.000316125860373648,
+      "loss": 1.1152,
+      "mean_token_accuracy": 0.7333301523700356,
+      "num_tokens": 496731129.0,
+      "step": 2840
+    },
+    {
+      "epoch": 0.7767122587678209,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.0003154235145385588,
+      "loss": 1.0788,
+      "mean_token_accuracy": 0.7434908222407103,
+      "num_tokens": 498416929.0,
+      "step": 2850
+    },
+    {
+      "epoch": 0.7794375649389361,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.0003147211687034696,
+      "loss": 1.065,
+      "mean_token_accuracy": 0.7451503177173435,
+      "num_tokens": 500131030.0,
+      "step": 2860
+    },
+    {
+      "epoch": 0.7821628711100512,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0003140188228683804,
+      "loss": 1.0668,
+      "mean_token_accuracy": 0.7436349290423095,
+      "num_tokens": 501860254.0,
+      "step": 2870
+    },
+    {
+      "epoch": 0.7848881772811664,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.0003133164770332912,
+      "loss": 1.1311,
+      "mean_token_accuracy": 0.7318741642870009,
+      "num_tokens": 503599791.0,
+      "step": 2880
+    },
+    {
+      "epoch": 0.7876134834522815,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.000312614131198202,
+      "loss": 1.1602,
+      "mean_token_accuracy": 0.7279526983387768,
+      "num_tokens": 505277304.0,
+      "step": 2890
+    },
+    {
+      "epoch": 0.7903387896233968,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003119117853631128,
+      "loss": 1.1026,
+      "mean_token_accuracy": 0.7374209313653409,
+      "num_tokens": 507010506.0,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7930640957945119,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00031120943952802363,
+      "loss": 1.0825,
+      "mean_token_accuracy": 0.7415086347609758,
+      "num_tokens": 508717678.0,
+      "step": 2910
+    },
+    {
+      "epoch": 0.7957894019656271,
+      "grad_norm": 0.326171875,
+      "learning_rate": 0.0003105070936929344,
+      "loss": 1.0521,
+      "mean_token_accuracy": 0.7478948015719652,
+      "num_tokens": 510478982.0,
+      "step": 2920
+    },
+    {
+      "epoch": 0.7985147081367422,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.00030980474785784526,
+      "loss": 1.059,
+      "mean_token_accuracy": 0.7448375985957683,
+      "num_tokens": 512223492.0,
+      "step": 2930
+    },
+    {
+      "epoch": 0.8012400143078574,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.000309102402022756,
+      "loss": 1.0837,
+      "mean_token_accuracy": 0.7406654883176088,
+      "num_tokens": 513950826.0,
+      "step": 2940
+    },
+    {
+      "epoch": 0.8039653204789725,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0003084000561876668,
+      "loss": 1.1186,
+      "mean_token_accuracy": 0.7354094261303544,
+      "num_tokens": 515747512.0,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8066906266500877,
+      "grad_norm": 0.318359375,
+      "learning_rate": 0.0003076977103525776,
+      "loss": 1.0841,
+      "mean_token_accuracy": 0.7409739443100989,
+      "num_tokens": 517564408.0,
+      "step": 2960
+    },
+    {
+      "epoch": 0.8094159328212028,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0003069953645174884,
+      "loss": 1.0163,
+      "mean_token_accuracy": 0.753525473922491,
+      "num_tokens": 519335113.0,
+      "step": 2970
+    },
+    {
+      "epoch": 0.812141238992318,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.00030629301868239923,
+      "loss": 1.0592,
+      "mean_token_accuracy": 0.7457088253460824,
+      "num_tokens": 521052694.0,
+      "step": 2980
+    },
+    {
+      "epoch": 0.8148665451634332,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.00030559067284731,
+      "loss": 1.1095,
+      "mean_token_accuracy": 0.7366851487196981,
+      "num_tokens": 522838886.0,
+      "step": 2990
+    },
+    {
+      "epoch": 0.8175918513345484,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0003048883270122208,
+      "loss": 1.0807,
+      "mean_token_accuracy": 0.7409872910007834,
+      "num_tokens": 524659688.0,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8203171575056635,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.0003041859811771316,
+      "loss": 1.0652,
+      "mean_token_accuracy": 0.7444087780080736,
+      "num_tokens": 526428828.0,
+      "step": 3010
+    },
+    {
+      "epoch": 0.8230424636767787,
+      "grad_norm": 0.349609375,
+      "learning_rate": 0.00030348363534204244,
+      "loss": 1.0946,
+      "mean_token_accuracy": 0.737694866489619,
+      "num_tokens": 528178357.0,
+      "step": 3020
+    },
+    {
+      "epoch": 0.8257677698478938,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0003027812895069532,
+      "loss": 1.0771,
+      "mean_token_accuracy": 0.7418748954311013,
+      "num_tokens": 529968661.0,
+      "step": 3030
+    },
+    {
+      "epoch": 0.828493076019009,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.00030207894367186407,
+      "loss": 1.0785,
+      "mean_token_accuracy": 0.7434822773560882,
+      "num_tokens": 531735846.0,
+      "step": 3040
+    },
+    {
+      "epoch": 0.8312183821901241,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00030137659783677483,
+      "loss": 1.1008,
+      "mean_token_accuracy": 0.7383057860657573,
+      "num_tokens": 533482605.0,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8339436883612393,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00030067425200168565,
+      "loss": 1.0853,
+      "mean_token_accuracy": 0.7413273256272077,
+      "num_tokens": 535172117.0,
+      "step": 3060
+    },
+    {
+      "epoch": 0.8366689945323545,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0002999719061665964,
+      "loss": 1.0709,
+      "mean_token_accuracy": 0.7434915580786765,
+      "num_tokens": 536922982.0,
+      "step": 3070
+    },
+    {
+      "epoch": 0.8393943007034697,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0002992695603315073,
+      "loss": 1.0856,
+      "mean_token_accuracy": 0.740437910053879,
+      "num_tokens": 538697561.0,
+      "step": 3080
+    },
+    {
+      "epoch": 0.8421196068745849,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.00029856721449641804,
+      "loss": 1.0881,
+      "mean_token_accuracy": 0.7398839842528104,
+      "num_tokens": 540498809.0,
+      "step": 3090
+    },
+    {
+      "epoch": 0.8448449130457,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.00029786486866132886,
+      "loss": 1.0505,
+      "mean_token_accuracy": 0.74623784115538,
+      "num_tokens": 542220573.0,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8475702192168152,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0002971625228262396,
+      "loss": 1.0689,
+      "mean_token_accuracy": 0.7432276338338852,
+      "num_tokens": 543995979.0,
+      "step": 3110
+    },
+    {
+      "epoch": 0.8502955253879303,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0002964601769911505,
+      "loss": 1.1001,
+      "mean_token_accuracy": 0.7364274661056698,
+      "num_tokens": 545762046.0,
+      "step": 3120
+    },
+    {
+      "epoch": 0.8530208315590455,
+      "grad_norm": 0.306640625,
+      "learning_rate": 0.00029575783115606125,
+      "loss": 1.0657,
+      "mean_token_accuracy": 0.7446419023908675,
+      "num_tokens": 547563120.0,
+      "step": 3130
+    },
+    {
+      "epoch": 0.8557461377301606,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00029505548532097206,
+      "loss": 1.0868,
+      "mean_token_accuracy": 0.7397670422680676,
+      "num_tokens": 549276605.0,
+      "step": 3140
+    },
+    {
+      "epoch": 0.8584714439012758,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0002943531394858829,
+      "loss": 1.0652,
+      "mean_token_accuracy": 0.7441672122105956,
+      "num_tokens": 550991479.0,
+      "step": 3150
+    },
+    {
+      "epoch": 0.8611967500723909,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0002936507936507937,
+      "loss": 1.0578,
+      "mean_token_accuracy": 0.746835071220994,
+      "num_tokens": 552777662.0,
+      "step": 3160
+    },
+    {
+      "epoch": 0.8639220562435062,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.00029294844781570446,
+      "loss": 1.0466,
+      "mean_token_accuracy": 0.7476489477790892,
+      "num_tokens": 554539356.0,
+      "step": 3170
+    },
+    {
+      "epoch": 0.8666473624146213,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.0002922461019806152,
+      "loss": 1.1222,
+      "mean_token_accuracy": 0.733480770047754,
+      "num_tokens": 556212548.0,
+      "step": 3180
+    },
+    {
+      "epoch": 0.8693726685857365,
+      "grad_norm": 0.361328125,
+      "learning_rate": 0.0002915437561455261,
+      "loss": 1.0835,
+      "mean_token_accuracy": 0.7399150485172867,
+      "num_tokens": 557950461.0,
+      "step": 3190
+    },
+    {
+      "epoch": 0.8720979747568516,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.00029084141031043685,
+      "loss": 1.0566,
+      "mean_token_accuracy": 0.7452725498005748,
+      "num_tokens": 559715926.0,
+      "step": 3200
+    },
+    {
+      "epoch": 0.8748232809279668,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.00029013906447534766,
+      "loss": 1.0807,
+      "mean_token_accuracy": 0.7420402772724628,
+      "num_tokens": 561479394.0,
+      "step": 3210
+    },
+    {
+      "epoch": 0.8775485870990819,
+      "grad_norm": 0.3359375,
+      "learning_rate": 0.0002894367186402584,
+      "loss": 1.0717,
+      "mean_token_accuracy": 0.7428519351407885,
+      "num_tokens": 563271872.0,
+      "step": 3220
+    },
+    {
+      "epoch": 0.8802738932701971,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0002887343728051693,
+      "loss": 1.0623,
+      "mean_token_accuracy": 0.745782736968249,
+      "num_tokens": 565002210.0,
+      "step": 3230
+    },
+    {
+      "epoch": 0.8829991994413122,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00028803202697008006,
+      "loss": 0.9654,
+      "mean_token_accuracy": 0.7632681075483561,
+      "num_tokens": 566740287.0,
+      "step": 3240
+    },
+    {
+      "epoch": 0.8857245056124274,
+      "grad_norm": 0.30859375,
+      "learning_rate": 0.00028732968113499087,
+      "loss": 1.0976,
+      "mean_token_accuracy": 0.7386646059341728,
+      "num_tokens": 568560935.0,
+      "step": 3250
+    },
+    {
+      "epoch": 0.8884498117835425,
+      "grad_norm": 0.384765625,
+      "learning_rate": 0.0002866273352999017,
+      "loss": 1.0843,
+      "mean_token_accuracy": 0.7406178612262011,
+      "num_tokens": 570290159.0,
+      "step": 3260
+    },
+    {
+      "epoch": 0.8911751179546578,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0002859249894648125,
+      "loss": 1.0829,
+      "mean_token_accuracy": 0.740851585380733,
+      "num_tokens": 572076704.0,
+      "step": 3270
+    },
+    {
+      "epoch": 0.8939004241257729,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.00028522264362972326,
+      "loss": 1.1033,
+      "mean_token_accuracy": 0.7380231520161032,
+      "num_tokens": 573841975.0,
+      "step": 3280
+    },
+    {
+      "epoch": 0.8966257302968881,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.0002845202977946341,
+      "loss": 1.0641,
+      "mean_token_accuracy": 0.7450438223779201,
+      "num_tokens": 575587035.0,
+      "step": 3290
+    },
+    {
+      "epoch": 0.8993510364680032,
+      "grad_norm": 0.318359375,
+      "learning_rate": 0.0002838179519595449,
+      "loss": 1.0663,
+      "mean_token_accuracy": 0.7433103552088142,
+      "num_tokens": 577285852.0,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9020763426391184,
+      "grad_norm": 0.349609375,
+      "learning_rate": 0.0002831156061244557,
+      "loss": 1.0831,
+      "mean_token_accuracy": 0.7410020404495299,
+      "num_tokens": 579085053.0,
+      "step": 3310
+    },
+    {
+      "epoch": 0.9048016488102335,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.00028241326028936647,
+      "loss": 1.0401,
+      "mean_token_accuracy": 0.7481556948274374,
+      "num_tokens": 580874041.0,
+      "step": 3320
+    },
+    {
+      "epoch": 0.9075269549813487,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0002817109144542773,
+      "loss": 1.052,
+      "mean_token_accuracy": 0.7451382425613702,
+      "num_tokens": 582616757.0,
+      "step": 3330
+    },
+    {
+      "epoch": 0.9102522611524638,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0002810085686191881,
+      "loss": 1.0498,
+      "mean_token_accuracy": 0.7470538474619388,
+      "num_tokens": 584370810.0,
+      "step": 3340
+    },
+    {
+      "epoch": 0.912977567323579,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0002803062227840989,
+      "loss": 1.0599,
+      "mean_token_accuracy": 0.7450041811913252,
+      "num_tokens": 586143330.0,
+      "step": 3350
+    },
+    {
+      "epoch": 0.9157028734946941,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0002796038769490097,
+      "loss": 1.0129,
+      "mean_token_accuracy": 0.753754030354321,
+      "num_tokens": 587851884.0,
+      "step": 3360
+    },
+    {
+      "epoch": 0.9184281796658094,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.00027890153111392055,
+      "loss": 1.0408,
+      "mean_token_accuracy": 0.7472209732979536,
+      "num_tokens": 589601267.0,
+      "step": 3370
+    },
+    {
+      "epoch": 0.9211534858369245,
+      "grad_norm": 0.388671875,
+      "learning_rate": 0.0002781991852788313,
+      "loss": 1.0909,
+      "mean_token_accuracy": 0.7383921964094042,
+      "num_tokens": 591321205.0,
+      "step": 3380
+    },
+    {
+      "epoch": 0.9238787920080397,
+      "grad_norm": 0.291015625,
+      "learning_rate": 0.0002774968394437421,
+      "loss": 1.0126,
+      "mean_token_accuracy": 0.7546688699163496,
+      "num_tokens": 593110103.0,
+      "step": 3390
+    },
+    {
+      "epoch": 0.9266040981791548,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0002767944936086529,
+      "loss": 1.0781,
+      "mean_token_accuracy": 0.7418509462848306,
+      "num_tokens": 594823039.0,
+      "step": 3400
+    },
+    {
+      "epoch": 0.92932940435027,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0002760921477735637,
+      "loss": 1.0656,
+      "mean_token_accuracy": 0.7450129883363843,
+      "num_tokens": 596625643.0,
+      "step": 3410
+    },
+    {
+      "epoch": 0.9320547105213851,
+      "grad_norm": 0.3671875,
+      "learning_rate": 0.0002753898019384745,
+      "loss": 1.0814,
+      "mean_token_accuracy": 0.7387082081288099,
+      "num_tokens": 598409404.0,
+      "step": 3420
+    },
+    {
+      "epoch": 0.9347800166925003,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0002746874561033853,
+      "loss": 1.0619,
+      "mean_token_accuracy": 0.7441019948571921,
+      "num_tokens": 600131817.0,
+      "step": 3430
+    },
+    {
+      "epoch": 0.9375053228636154,
+      "grad_norm": 0.310546875,
+      "learning_rate": 0.0002739851102682961,
+      "loss": 1.0625,
+      "mean_token_accuracy": 0.7441184409894049,
+      "num_tokens": 601867823.0,
+      "step": 3440
+    },
+    {
+      "epoch": 0.9402306290347306,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.0002732827644332069,
+      "loss": 1.0346,
+      "mean_token_accuracy": 0.7519477549009025,
+      "num_tokens": 603591263.0,
+      "step": 3450
+    },
+    {
+      "epoch": 0.9429559352058458,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.0002725804185981177,
+      "loss": 1.0756,
+      "mean_token_accuracy": 0.7429594827815891,
+      "num_tokens": 605400422.0,
+      "step": 3460
+    },
+    {
+      "epoch": 0.945681241376961,
+      "grad_norm": 0.32421875,
+      "learning_rate": 0.0002718780727630285,
+      "loss": 1.0677,
+      "mean_token_accuracy": 0.7448620780371129,
+      "num_tokens": 607180330.0,
+      "step": 3470
+    },
+    {
+      "epoch": 0.9484065475480761,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.00027117572692793936,
+      "loss": 1.0215,
+      "mean_token_accuracy": 0.7525013948790729,
+      "num_tokens": 608937194.0,
+      "step": 3480
+    },
+    {
+      "epoch": 0.9511318537191913,
+      "grad_norm": 0.54296875,
+      "learning_rate": 0.0002704733810928501,
+      "loss": 1.0707,
+      "mean_token_accuracy": 0.7420834172517061,
+      "num_tokens": 610676168.0,
+      "step": 3490
+    },
+    {
+      "epoch": 0.9538571598903064,
+      "grad_norm": 0.30859375,
+      "learning_rate": 0.00026977103525776093,
+      "loss": 1.0088,
+      "mean_token_accuracy": 0.7546906681731343,
+      "num_tokens": 612491555.0,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9565824660614216,
+      "grad_norm": 0.36328125,
+      "learning_rate": 0.0002690686894226717,
+      "loss": 1.0682,
+      "mean_token_accuracy": 0.7417504930868745,
+      "num_tokens": 614310820.0,
+      "step": 3510
+    },
+    {
+      "epoch": 0.9593077722325367,
+      "grad_norm": 0.353515625,
+      "learning_rate": 0.00026836634358758256,
+      "loss": 1.0792,
+      "mean_token_accuracy": 0.7415831151418388,
+      "num_tokens": 616071087.0,
+      "step": 3520
+    },
+    {
+      "epoch": 0.9620330784036519,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.0002676639977524933,
+      "loss": 1.0179,
+      "mean_token_accuracy": 0.7520351181738079,
+      "num_tokens": 617769166.0,
+      "step": 3530
+    },
+    {
+      "epoch": 0.964758384574767,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00026696165191740414,
+      "loss": 1.0445,
+      "mean_token_accuracy": 0.7471835752017796,
+      "num_tokens": 619538232.0,
+      "step": 3540
+    },
+    {
+      "epoch": 0.9674836907458823,
+      "grad_norm": 0.29296875,
+      "learning_rate": 0.0002662593060823149,
+      "loss": 1.0407,
+      "mean_token_accuracy": 0.74965282054618,
+      "num_tokens": 621303314.0,
+      "step": 3550
+    },
+    {
+      "epoch": 0.9702089969169974,
+      "grad_norm": 0.298828125,
+      "learning_rate": 0.00026555696024722577,
+      "loss": 1.027,
+      "mean_token_accuracy": 0.751960375905037,
+      "num_tokens": 623100776.0,
+      "step": 3560
+    },
+    {
+      "epoch": 0.9729343030881126,
+      "grad_norm": 0.3125,
+      "learning_rate": 0.00026485461441213653,
+      "loss": 1.096,
+      "mean_token_accuracy": 0.736579249240458,
+      "num_tokens": 624821413.0,
+      "step": 3570
+    },
+    {
+      "epoch": 0.9756596092592277,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.00026415226857704735,
+      "loss": 1.0511,
+      "mean_token_accuracy": 0.7452825047075748,
+      "num_tokens": 626628545.0,
+      "step": 3580
+    },
+    {
+      "epoch": 0.9783849154303429,
+      "grad_norm": 0.318359375,
+      "learning_rate": 0.00026344992274195816,
+      "loss": 1.0388,
+      "mean_token_accuracy": 0.749433082062751,
+      "num_tokens": 628425036.0,
+      "step": 3590
+    },
+    {
+      "epoch": 0.981110221601458,
+      "grad_norm": 0.30859375,
+      "learning_rate": 0.000262747576906869,
+      "loss": 1.0046,
+      "mean_token_accuracy": 0.7535642351955175,
+      "num_tokens": 630092739.0,
+      "step": 3600
+    },
+    {
+      "epoch": 0.9838355277725732,
+      "grad_norm": 0.375,
+      "learning_rate": 0.00026204523107177974,
+      "loss": 1.0579,
+      "mean_token_accuracy": 0.7460371427237987,
+      "num_tokens": 631791605.0,
+      "step": 3610
+    },
+    {
+      "epoch": 0.9865608339436883,
+      "grad_norm": 0.73046875,
+      "learning_rate": 0.00026134288523669056,
+      "loss": 1.0452,
+      "mean_token_accuracy": 0.7458532767370343,
+      "num_tokens": 633554055.0,
+      "step": 3620
+    },
+    {
+      "epoch": 0.9892861401148035,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.00026064053940160137,
+      "loss": 1.0193,
+      "mean_token_accuracy": 0.7518707301467658,
+      "num_tokens": 635269789.0,
+      "step": 3630
+    },
+    {
+      "epoch": 0.9920114462859186,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.00025993819356651213,
+      "loss": 1.0367,
+      "mean_token_accuracy": 0.7495086012408138,
+      "num_tokens": 637078092.0,
+      "step": 3640
+    },
+    {
+      "epoch": 0.9947367524570339,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.00025923584773142295,
+      "loss": 1.0075,
+      "mean_token_accuracy": 0.7556772115640342,
+      "num_tokens": 638813927.0,
+      "step": 3650
+    },
+    {
+      "epoch": 0.997462058628149,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.0002585335018963337,
+      "loss": 1.0366,
+      "mean_token_accuracy": 0.7495333584956825,
+      "num_tokens": 640594007.0,
+      "step": 3660
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.1640625,
+      "learning_rate": 0.0002578311560612446,
+      "loss": 0.9358,
+      "mean_token_accuracy": 0.7545788142105077,
+      "num_tokens": 642209396.0,
+      "step": 3670
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7340,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500.0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.068172081287866e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3670/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f00d55cafeda09e6936676fbcbeac40ed9bcedc85bbbc85cdf29a53e944f42c
+size 6072

checkpoint-7340/chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-7340/config.json

@@ -0,0 +1,132 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_factor": 0.5,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": -1,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": false,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 16,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 4,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 256,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 128,
+  "vocab_size": 128256
+}

checkpoint-7340/configuration_hymba.py

@@ -0,0 +1,122 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            attn_factor=0.5,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.attn_factor = attn_factor
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+        
+        self.pure_attn = pure_attn
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

checkpoint-7340/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

checkpoint-7340/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:538afd1aad45266f8557c2a29f8496a376e6cfc1f4a9e0d2c44519448b78e678
+size 2471642200

checkpoint-7340/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e68a14e261ba2c7432b5cdc578e0e6750c8fc57e43598ff50ebb3d04447e417d
+size 4943354422

checkpoint-7340/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-7340/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332d348ed4d6c605735836627555000674392b75beb40082a7a320d4f9f70304
+size 1064

checkpoint-7340/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-7340/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-7340/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-7340/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f00d55cafeda09e6936676fbcbeac40ed9bcedc85bbbc85cdf29a53e944f42c
+size 6072

config.json

@@ -0,0 +1,132 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_factor": 0.5,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": -1,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": false,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 16,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 4,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 256,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 128,
+  "vocab_size": 128256
+}

configuration_hymba.py

@@ -0,0 +1,122 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            attn_factor=0.5,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.attn_factor = attn_factor
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+        
+        self.pure_attn = pure_attn
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:538afd1aad45266f8557c2a29f8496a376e6cfc1f4a9e0d2c44519448b78e678
+size 2471642200

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f00d55cafeda09e6936676fbcbeac40ed9bcedc85bbbc85cdf29a53e944f42c
+size 6072