First commit

README.md

@@ -1,3 +1,74 @@
----
-license: mit
----
+---
+license: mit
+
+---
+
+# Model Card for SciLitLLM1.5
+
+SciLitLLM1.5 adapts a general large language model for effective scientific literature understanding. Starting from the Qwen2.5-7B/14B model, SciLitLLM1.5-7B/14B goes through a hybrid strategy that integrates continual pre-training (CPT) and supervised fine-tuning (SFT), to simultaneously infuse scientific domain knowledge and enhance instruction-following capabilities for domain-specific tasks.
+
+In this process, we identify two key challenges: (1) constructing high-quality CPT corpora, and (2) generating diverse SFT instructions. We address these challenges through a meticulous pipeline, including PDF text extraction, parsing content error correction, quality filtering, and synthetic instruction creation.
+
+Applying this strategy, we present SciLitLLM-7B and 14B, specialized in scientific literature understanding, which demonstrates promising performance on scientific literature understanding benchmarks. 
+
+We observe promising performance enhancements, **with an average improvement of 4.0\% on SciAssess and 10.1% on SciRIFF, compared to the leading LLMs under 10B parameters**. Notably, **SciLitLLM-7B even outperforms Llama3.1 and Qwen2.5 with 70B parameters on SciRIFF**.  Additionally, SciLitLLM-14B achieves leading results on both benchmarks, surpassing other open-source LLMs. Further ablation studies demonstrate the effectiveness of each module in our pipeline.
+
+See the [paper](https://arxiv.org/abs/2408.15545) for more details and [github](https://github.com/dptech-corp/Uni-SMART) for data processing codes.
+
+## Requirements
+
+Since SciLitLLM is based on Qwen2.5, we advise you to install `transformers>=4.37.0`, or you might encounter the following error:
+
+```
+KeyError: 'qwen2'
+```
+
+## Quickstart
+
+Here provides a code snippet with `apply_chat_template` to show you how to load the tokenizer and model and how to generate contents.
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+device = "cuda" # the device to load the model onto
+model = AutoModelForCausalLM.from_pretrained(
+    "Uni-SMART/SciLitLLM-1.5/7B/",
+    torch_dtype="auto",
+    device_map="auto"
+)
+tokenizer = AutoTokenizer.from_pretrained("Uni-SMART/SciLitLLM-1.5/7B/")
+prompt = "Can you summarize this article for me?\n <ARTICLE>"
+messages = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": prompt}
+]
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+model_inputs = tokenizer([text], return_tensors="pt").to(device)
+generated_ids = model.generate(
+    model_inputs.input_ids,
+    max_new_tokens=512
+)
+generated_ids = [
+    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+]
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+```
+
+## Citation
+
+If you find our work helpful, feel free to give us a cite.
+
+```
+@misc{li2024scilitllmadaptllmsscientific,
+      title={SciLitLLM: How to Adapt LLMs for Scientific Literature Understanding}, 
+      author={Sihang Li and Jin Huang and Jiaxi Zhuang and Yaorui Shi and Xiaochen Cai and Mingjun Xu and Xiang Wang and Linfeng Zhang and Guolin Ke and Hengxing Cai},
+      year={2024},
+      eprint={2408.15545},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2408.15545}, 
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "/mnt/vepfs/fs_unifinder/SciLLM/models/Qwen/Qwen2.5-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "original_max_position_embeddings": 32768,
+    "type": "yarn"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.43.4"
+}

pytorch_model-00001-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d07aeb2516c32e8cf44a890e0f6efddd9e176d3f3bd65d5a9367d21e4bc86ef
+size 4877684182

pytorch_model-00002-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01a53abd3954e835f957b7248dbee63c477ebd433318dca2c78c786ebae77297
+size 4932778600

pytorch_model-00003-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1bbe7c59e2784205a05cdd6a83eb74cc75a3f422f417b4e148c08dfb93d0449
+size 4330890394

pytorch_model-00004-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cee19a5ab363f29811eacda6fd268f9012bcede636cebdd96c6be6309ad48287
+size 1089996165

pytorch_model.bin.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00004-of-00004.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.k_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.q_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.v_proj.bias": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.self_attn.k_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.q_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.v_proj.bias": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.k_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.q_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.v_proj.bias": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.norm.weight": "pytorch_model-00003-of-00004.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,2315 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9996167113836719,
+  "eval_steps": 326,
+  "global_step": 326,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0030663089306247606,
+      "grad_norm": 26.20577049255371,
+      "learning_rate": 1.5151515151515152e-07,
+      "loss": 2.0736,
+      "step": 1
+    },
+    {
+      "epoch": 0.006132617861249521,
+      "grad_norm": 24.32123565673828,
+      "learning_rate": 3.0303030303030305e-07,
+      "loss": 2.1049,
+      "step": 2
+    },
+    {
+      "epoch": 0.009198926791874281,
+      "grad_norm": 23.082963943481445,
+      "learning_rate": 4.5454545454545457e-07,
+      "loss": 1.8755,
+      "step": 3
+    },
+    {
+      "epoch": 0.012265235722499043,
+      "grad_norm": 26.788209915161133,
+      "learning_rate": 6.060606060606061e-07,
+      "loss": 2.1995,
+      "step": 4
+    },
+    {
+      "epoch": 0.015331544653123802,
+      "grad_norm": 21.350114822387695,
+      "learning_rate": 7.575757575757576e-07,
+      "loss": 2.1673,
+      "step": 5
+    },
+    {
+      "epoch": 0.018397853583748562,
+      "grad_norm": 28.563199996948242,
+      "learning_rate": 9.090909090909091e-07,
+      "loss": 2.2216,
+      "step": 6
+    },
+    {
+      "epoch": 0.021464162514373324,
+      "grad_norm": 19.89320945739746,
+      "learning_rate": 1.0606060606060608e-06,
+      "loss": 1.8264,
+      "step": 7
+    },
+    {
+      "epoch": 0.024530471444998085,
+      "grad_norm": 20.160602569580078,
+      "learning_rate": 1.2121212121212122e-06,
+      "loss": 2.0688,
+      "step": 8
+    },
+    {
+      "epoch": 0.027596780375622843,
+      "grad_norm": 21.4703369140625,
+      "learning_rate": 1.3636363636363636e-06,
+      "loss": 2.0488,
+      "step": 9
+    },
+    {
+      "epoch": 0.030663089306247605,
+      "grad_norm": 32.90473175048828,
+      "learning_rate": 1.5151515151515152e-06,
+      "loss": 2.386,
+      "step": 10
+    },
+    {
+      "epoch": 0.03372939823687236,
+      "grad_norm": 20.769977569580078,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 1.6769,
+      "step": 11
+    },
+    {
+      "epoch": 0.036795707167497124,
+      "grad_norm": 22.5015811920166,
+      "learning_rate": 1.8181818181818183e-06,
+      "loss": 1.3397,
+      "step": 12
+    },
+    {
+      "epoch": 0.039862016098121886,
+      "grad_norm": 18.01841163635254,
+      "learning_rate": 1.96969696969697e-06,
+      "loss": 1.2202,
+      "step": 13
+    },
+    {
+      "epoch": 0.04292832502874665,
+      "grad_norm": 13.076176643371582,
+      "learning_rate": 2.1212121212121216e-06,
+      "loss": 1.0673,
+      "step": 14
+    },
+    {
+      "epoch": 0.04599463395937141,
+      "grad_norm": 7.562614917755127,
+      "learning_rate": 2.2727272727272728e-06,
+      "loss": 1.022,
+      "step": 15
+    },
+    {
+      "epoch": 0.04906094288999617,
+      "grad_norm": 5.768344402313232,
+      "learning_rate": 2.4242424242424244e-06,
+      "loss": 0.9034,
+      "step": 16
+    },
+    {
+      "epoch": 0.052127251820620925,
+      "grad_norm": 8.617671012878418,
+      "learning_rate": 2.575757575757576e-06,
+      "loss": 0.8786,
+      "step": 17
+    },
+    {
+      "epoch": 0.055193560751245686,
+      "grad_norm": 5.182284355163574,
+      "learning_rate": 2.7272727272727272e-06,
+      "loss": 0.854,
+      "step": 18
+    },
+    {
+      "epoch": 0.05825986968187045,
+      "grad_norm": 4.750306129455566,
+      "learning_rate": 2.8787878787878793e-06,
+      "loss": 0.8039,
+      "step": 19
+    },
+    {
+      "epoch": 0.06132617861249521,
+      "grad_norm": 2.5204381942749023,
+      "learning_rate": 3.0303030303030305e-06,
+      "loss": 0.7402,
+      "step": 20
+    },
+    {
+      "epoch": 0.06439248754311996,
+      "grad_norm": 3.134700298309326,
+      "learning_rate": 3.181818181818182e-06,
+      "loss": 0.7149,
+      "step": 21
+    },
+    {
+      "epoch": 0.06745879647374473,
+      "grad_norm": 2.9186689853668213,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.7597,
+      "step": 22
+    },
+    {
+      "epoch": 0.07052510540436949,
+      "grad_norm": 2.0178744792938232,
+      "learning_rate": 3.4848484848484854e-06,
+      "loss": 0.6614,
+      "step": 23
+    },
+    {
+      "epoch": 0.07359141433499425,
+      "grad_norm": 2.2368788719177246,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.749,
+      "step": 24
+    },
+    {
+      "epoch": 0.07665772326561901,
+      "grad_norm": 1.8660510778427124,
+      "learning_rate": 3.7878787878787882e-06,
+      "loss": 0.7469,
+      "step": 25
+    },
+    {
+      "epoch": 0.07972403219624377,
+      "grad_norm": 1.5830739736557007,
+      "learning_rate": 3.93939393939394e-06,
+      "loss": 0.7101,
+      "step": 26
+    },
+    {
+      "epoch": 0.08279034112686853,
+      "grad_norm": 1.8778492212295532,
+      "learning_rate": 4.0909090909090915e-06,
+      "loss": 0.6604,
+      "step": 27
+    },
+    {
+      "epoch": 0.0858566500574933,
+      "grad_norm": 2.057135820388794,
+      "learning_rate": 4.242424242424243e-06,
+      "loss": 0.721,
+      "step": 28
+    },
+    {
+      "epoch": 0.08892295898811806,
+      "grad_norm": 1.7224825620651245,
+      "learning_rate": 4.393939393939394e-06,
+      "loss": 0.7502,
+      "step": 29
+    },
+    {
+      "epoch": 0.09198926791874282,
+      "grad_norm": 1.4657948017120361,
+      "learning_rate": 4.5454545454545455e-06,
+      "loss": 0.7054,
+      "step": 30
+    },
+    {
+      "epoch": 0.09505557684936758,
+      "grad_norm": 1.4230220317840576,
+      "learning_rate": 4.696969696969698e-06,
+      "loss": 0.6655,
+      "step": 31
+    },
+    {
+      "epoch": 0.09812188577999234,
+      "grad_norm": 1.6535286903381348,
+      "learning_rate": 4.848484848484849e-06,
+      "loss": 0.7163,
+      "step": 32
+    },
+    {
+      "epoch": 0.10118819471061709,
+      "grad_norm": 1.6131975650787354,
+      "learning_rate": 5e-06,
+      "loss": 0.6682,
+      "step": 33
+    },
+    {
+      "epoch": 0.10425450364124185,
+      "grad_norm": 2.1076509952545166,
+      "learning_rate": 5.151515151515152e-06,
+      "loss": 0.6938,
+      "step": 34
+    },
+    {
+      "epoch": 0.10732081257186661,
+      "grad_norm": 1.5259658098220825,
+      "learning_rate": 5.303030303030303e-06,
+      "loss": 0.6784,
+      "step": 35
+    },
+    {
+      "epoch": 0.11038712150249137,
+      "grad_norm": 2.1185150146484375,
+      "learning_rate": 5.4545454545454545e-06,
+      "loss": 0.6916,
+      "step": 36
+    },
+    {
+      "epoch": 0.11345343043311613,
+      "grad_norm": 1.6294972896575928,
+      "learning_rate": 5.606060606060606e-06,
+      "loss": 0.7065,
+      "step": 37
+    },
+    {
+      "epoch": 0.1165197393637409,
+      "grad_norm": 1.500267505645752,
+      "learning_rate": 5.7575757575757586e-06,
+      "loss": 0.661,
+      "step": 38
+    },
+    {
+      "epoch": 0.11958604829436566,
+      "grad_norm": 1.4696543216705322,
+      "learning_rate": 5.90909090909091e-06,
+      "loss": 0.6059,
+      "step": 39
+    },
+    {
+      "epoch": 0.12265235722499042,
+      "grad_norm": 1.445040225982666,
+      "learning_rate": 6.060606060606061e-06,
+      "loss": 0.6724,
+      "step": 40
+    },
+    {
+      "epoch": 0.12571866615561517,
+      "grad_norm": 1.3363935947418213,
+      "learning_rate": 6.212121212121213e-06,
+      "loss": 0.6636,
+      "step": 41
+    },
+    {
+      "epoch": 0.12878497508623993,
+      "grad_norm": 1.6588456630706787,
+      "learning_rate": 6.363636363636364e-06,
+      "loss": 0.7041,
+      "step": 42
+    },
+    {
+      "epoch": 0.1318512840168647,
+      "grad_norm": 1.551624059677124,
+      "learning_rate": 6.515151515151516e-06,
+      "loss": 0.6949,
+      "step": 43
+    },
+    {
+      "epoch": 0.13491759294748945,
+      "grad_norm": 1.353107213973999,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.657,
+      "step": 44
+    },
+    {
+      "epoch": 0.1379839018781142,
+      "grad_norm": 1.4454243183135986,
+      "learning_rate": 6.818181818181818e-06,
+      "loss": 0.636,
+      "step": 45
+    },
+    {
+      "epoch": 0.14105021080873897,
+      "grad_norm": 1.586327314376831,
+      "learning_rate": 6.969696969696971e-06,
+      "loss": 0.7216,
+      "step": 46
+    },
+    {
+      "epoch": 0.14411651973936374,
+      "grad_norm": 1.4303900003433228,
+      "learning_rate": 7.121212121212122e-06,
+      "loss": 0.6787,
+      "step": 47
+    },
+    {
+      "epoch": 0.1471828286699885,
+      "grad_norm": 1.769648790359497,
+      "learning_rate": 7.272727272727273e-06,
+      "loss": 0.5773,
+      "step": 48
+    },
+    {
+      "epoch": 0.15024913760061326,
+      "grad_norm": 2.123302459716797,
+      "learning_rate": 7.424242424242425e-06,
+      "loss": 0.6263,
+      "step": 49
+    },
+    {
+      "epoch": 0.15331544653123802,
+      "grad_norm": 1.216840386390686,
+      "learning_rate": 7.5757575757575764e-06,
+      "loss": 0.5872,
+      "step": 50
+    },
+    {
+      "epoch": 0.15638175546186278,
+      "grad_norm": 1.5153552293777466,
+      "learning_rate": 7.727272727272727e-06,
+      "loss": 0.594,
+      "step": 51
+    },
+    {
+      "epoch": 0.15944806439248754,
+      "grad_norm": 1.701780915260315,
+      "learning_rate": 7.87878787878788e-06,
+      "loss": 0.5753,
+      "step": 52
+    },
+    {
+      "epoch": 0.1625143733231123,
+      "grad_norm": 1.5358452796936035,
+      "learning_rate": 8.03030303030303e-06,
+      "loss": 0.5823,
+      "step": 53
+    },
+    {
+      "epoch": 0.16558068225373707,
+      "grad_norm": 1.4282408952713013,
+      "learning_rate": 8.181818181818183e-06,
+      "loss": 0.5633,
+      "step": 54
+    },
+    {
+      "epoch": 0.16864699118436183,
+      "grad_norm": 1.518804907798767,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.6381,
+      "step": 55
+    },
+    {
+      "epoch": 0.1717133001149866,
+      "grad_norm": 1.427336573600769,
+      "learning_rate": 8.484848484848486e-06,
+      "loss": 0.5919,
+      "step": 56
+    },
+    {
+      "epoch": 0.17477960904561135,
+      "grad_norm": 1.8106756210327148,
+      "learning_rate": 8.636363636363637e-06,
+      "loss": 0.617,
+      "step": 57
+    },
+    {
+      "epoch": 0.1778459179762361,
+      "grad_norm": 1.7282638549804688,
+      "learning_rate": 8.787878787878788e-06,
+      "loss": 0.5657,
+      "step": 58
+    },
+    {
+      "epoch": 0.18091222690686087,
+      "grad_norm": 1.22378671169281,
+      "learning_rate": 8.93939393939394e-06,
+      "loss": 0.6361,
+      "step": 59
+    },
+    {
+      "epoch": 0.18397853583748563,
+      "grad_norm": 1.6425446271896362,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": 0.5809,
+      "step": 60
+    },
+    {
+      "epoch": 0.1870448447681104,
+      "grad_norm": 1.671093225479126,
+      "learning_rate": 9.242424242424244e-06,
+      "loss": 0.6552,
+      "step": 61
+    },
+    {
+      "epoch": 0.19011115369873516,
+      "grad_norm": 1.3401716947555542,
+      "learning_rate": 9.393939393939396e-06,
+      "loss": 0.5941,
+      "step": 62
+    },
+    {
+      "epoch": 0.19317746262935992,
+      "grad_norm": 1.7498574256896973,
+      "learning_rate": 9.545454545454547e-06,
+      "loss": 0.7235,
+      "step": 63
+    },
+    {
+      "epoch": 0.19624377155998468,
+      "grad_norm": 1.510758876800537,
+      "learning_rate": 9.696969696969698e-06,
+      "loss": 0.6712,
+      "step": 64
+    },
+    {
+      "epoch": 0.19931008049060944,
+      "grad_norm": 1.496213674545288,
+      "learning_rate": 9.84848484848485e-06,
+      "loss": 0.5502,
+      "step": 65
+    },
+    {
+      "epoch": 0.20237638942123418,
+      "grad_norm": 1.8374212980270386,
+      "learning_rate": 1e-05,
+      "loss": 0.6098,
+      "step": 66
+    },
+    {
+      "epoch": 0.20544269835185894,
+      "grad_norm": 1.3000707626342773,
+      "learning_rate": 9.999997581367439e-06,
+      "loss": 0.6399,
+      "step": 67
+    },
+    {
+      "epoch": 0.2085090072824837,
+      "grad_norm": 1.5495225191116333,
+      "learning_rate": 9.999990325472093e-06,
+      "loss": 0.6518,
+      "step": 68
+    },
+    {
+      "epoch": 0.21157531621310846,
+      "grad_norm": 1.5361844301223755,
+      "learning_rate": 9.999978232320984e-06,
+      "loss": 0.5714,
+      "step": 69
+    },
+    {
+      "epoch": 0.21464162514373322,
+      "grad_norm": 1.9379122257232666,
+      "learning_rate": 9.99996130192581e-06,
+      "loss": 0.6144,
+      "step": 70
+    },
+    {
+      "epoch": 0.21770793407435798,
+      "grad_norm": 1.522295355796814,
+      "learning_rate": 9.999939534302951e-06,
+      "loss": 0.6434,
+      "step": 71
+    },
+    {
+      "epoch": 0.22077424300498275,
+      "grad_norm": 4.380992412567139,
+      "learning_rate": 9.999912929473465e-06,
+      "loss": 0.5859,
+      "step": 72
+    },
+    {
+      "epoch": 0.2238405519356075,
+      "grad_norm": 2.6046173572540283,
+      "learning_rate": 9.999881487463094e-06,
+      "loss": 0.6552,
+      "step": 73
+    },
+    {
+      "epoch": 0.22690686086623227,
+      "grad_norm": 1.627494215965271,
+      "learning_rate": 9.999845208302253e-06,
+      "loss": 0.5189,
+      "step": 74
+    },
+    {
+      "epoch": 0.22997316979685703,
+      "grad_norm": 2.6718268394470215,
+      "learning_rate": 9.999804092026043e-06,
+      "loss": 0.6102,
+      "step": 75
+    },
+    {
+      "epoch": 0.2330394787274818,
+      "grad_norm": 2.5320703983306885,
+      "learning_rate": 9.99975813867424e-06,
+      "loss": 0.6342,
+      "step": 76
+    },
+    {
+      "epoch": 0.23610578765810655,
+      "grad_norm": 2.0609052181243896,
+      "learning_rate": 9.999707348291306e-06,
+      "loss": 0.5861,
+      "step": 77
+    },
+    {
+      "epoch": 0.23917209658873131,
+      "grad_norm": 1.5791586637496948,
+      "learning_rate": 9.999651720926371e-06,
+      "loss": 0.6189,
+      "step": 78
+    },
+    {
+      "epoch": 0.24223840551935608,
+      "grad_norm": 1.4853404760360718,
+      "learning_rate": 9.999591256633258e-06,
+      "loss": 0.6027,
+      "step": 79
+    },
+    {
+      "epoch": 0.24530471444998084,
+      "grad_norm": 1.3395217657089233,
+      "learning_rate": 9.999525955470462e-06,
+      "loss": 0.6136,
+      "step": 80
+    },
+    {
+      "epoch": 0.2483710233806056,
+      "grad_norm": 1.511687994003296,
+      "learning_rate": 9.999455817501155e-06,
+      "loss": 0.6968,
+      "step": 81
+    },
+    {
+      "epoch": 0.25143733231123033,
+      "grad_norm": 2.3173654079437256,
+      "learning_rate": 9.9993808427932e-06,
+      "loss": 0.6092,
+      "step": 82
+    },
+    {
+      "epoch": 0.2545036412418551,
+      "grad_norm": 1.720035433769226,
+      "learning_rate": 9.999301031419124e-06,
+      "loss": 0.5702,
+      "step": 83
+    },
+    {
+      "epoch": 0.25756995017247986,
+      "grad_norm": 1.3266725540161133,
+      "learning_rate": 9.999216383456145e-06,
+      "loss": 0.589,
+      "step": 84
+    },
+    {
+      "epoch": 0.2606362591031046,
+      "grad_norm": 1.4458568096160889,
+      "learning_rate": 9.999126898986152e-06,
+      "loss": 0.6213,
+      "step": 85
+    },
+    {
+      "epoch": 0.2637025680337294,
+      "grad_norm": 1.1566760540008545,
+      "learning_rate": 9.999032578095722e-06,
+      "loss": 0.596,
+      "step": 86
+    },
+    {
+      "epoch": 0.26676887696435414,
+      "grad_norm": 1.462127685546875,
+      "learning_rate": 9.998933420876103e-06,
+      "loss": 0.5912,
+      "step": 87
+    },
+    {
+      "epoch": 0.2698351858949789,
+      "grad_norm": 1.3121938705444336,
+      "learning_rate": 9.998829427423224e-06,
+      "loss": 0.5855,
+      "step": 88
+    },
+    {
+      "epoch": 0.27290149482560366,
+      "grad_norm": 1.325122594833374,
+      "learning_rate": 9.998720597837698e-06,
+      "loss": 0.664,
+      "step": 89
+    },
+    {
+      "epoch": 0.2759678037562284,
+      "grad_norm": 1.325933575630188,
+      "learning_rate": 9.998606932224808e-06,
+      "loss": 0.6864,
+      "step": 90
+    },
+    {
+      "epoch": 0.2790341126868532,
+      "grad_norm": 1.3255521059036255,
+      "learning_rate": 9.998488430694524e-06,
+      "loss": 0.6137,
+      "step": 91
+    },
+    {
+      "epoch": 0.28210042161747795,
+      "grad_norm": 1.4992860555648804,
+      "learning_rate": 9.998365093361486e-06,
+      "loss": 0.592,
+      "step": 92
+    },
+    {
+      "epoch": 0.2851667305481027,
+      "grad_norm": 1.5969640016555786,
+      "learning_rate": 9.998236920345021e-06,
+      "loss": 0.59,
+      "step": 93
+    },
+    {
+      "epoch": 0.28823303947872747,
+      "grad_norm": 1.4171638488769531,
+      "learning_rate": 9.998103911769129e-06,
+      "loss": 0.5732,
+      "step": 94
+    },
+    {
+      "epoch": 0.29129934840935223,
+      "grad_norm": 1.3092645406723022,
+      "learning_rate": 9.99796606776249e-06,
+      "loss": 0.5886,
+      "step": 95
+    },
+    {
+      "epoch": 0.294365657339977,
+      "grad_norm": 1.6398749351501465,
+      "learning_rate": 9.997823388458462e-06,
+      "loss": 0.6358,
+      "step": 96
+    },
+    {
+      "epoch": 0.29743196627060176,
+      "grad_norm": 1.389186143875122,
+      "learning_rate": 9.997675873995077e-06,
+      "loss": 0.564,
+      "step": 97
+    },
+    {
+      "epoch": 0.3004982752012265,
+      "grad_norm": 1.499334454536438,
+      "learning_rate": 9.997523524515054e-06,
+      "loss": 0.6365,
+      "step": 98
+    },
+    {
+      "epoch": 0.3035645841318513,
+      "grad_norm": 1.5321111679077148,
+      "learning_rate": 9.99736634016578e-06,
+      "loss": 0.6761,
+      "step": 99
+    },
+    {
+      "epoch": 0.30663089306247604,
+      "grad_norm": 1.6426483392715454,
+      "learning_rate": 9.997204321099324e-06,
+      "loss": 0.6466,
+      "step": 100
+    },
+    {
+      "epoch": 0.3096972019931008,
+      "grad_norm": 1.648946762084961,
+      "learning_rate": 9.997037467472432e-06,
+      "loss": 0.6073,
+      "step": 101
+    },
+    {
+      "epoch": 0.31276351092372556,
+      "grad_norm": 1.2670505046844482,
+      "learning_rate": 9.996865779446527e-06,
+      "loss": 0.6228,
+      "step": 102
+    },
+    {
+      "epoch": 0.3158298198543503,
+      "grad_norm": 1.4200985431671143,
+      "learning_rate": 9.996689257187712e-06,
+      "loss": 0.6542,
+      "step": 103
+    },
+    {
+      "epoch": 0.3188961287849751,
+      "grad_norm": 1.3386104106903076,
+      "learning_rate": 9.99650790086676e-06,
+      "loss": 0.6249,
+      "step": 104
+    },
+    {
+      "epoch": 0.32196243771559985,
+      "grad_norm": 1.723291277885437,
+      "learning_rate": 9.996321710659125e-06,
+      "loss": 0.5767,
+      "step": 105
+    },
+    {
+      "epoch": 0.3250287466462246,
+      "grad_norm": 1.2609390020370483,
+      "learning_rate": 9.99613068674494e-06,
+      "loss": 0.5908,
+      "step": 106
+    },
+    {
+      "epoch": 0.32809505557684937,
+      "grad_norm": 1.5247840881347656,
+      "learning_rate": 9.995934829309011e-06,
+      "loss": 0.5468,
+      "step": 107
+    },
+    {
+      "epoch": 0.33116136450747413,
+      "grad_norm": 1.3155369758605957,
+      "learning_rate": 9.995734138540818e-06,
+      "loss": 0.5901,
+      "step": 108
+    },
+    {
+      "epoch": 0.3342276734380989,
+      "grad_norm": 1.3768904209136963,
+      "learning_rate": 9.995528614634524e-06,
+      "loss": 0.5992,
+      "step": 109
+    },
+    {
+      "epoch": 0.33729398236872365,
+      "grad_norm": 1.288352131843567,
+      "learning_rate": 9.995318257788962e-06,
+      "loss": 0.6146,
+      "step": 110
+    },
+    {
+      "epoch": 0.3403602912993484,
+      "grad_norm": 1.3756117820739746,
+      "learning_rate": 9.995103068207638e-06,
+      "loss": 0.5785,
+      "step": 111
+    },
+    {
+      "epoch": 0.3434266002299732,
+      "grad_norm": 1.3425506353378296,
+      "learning_rate": 9.994883046098745e-06,
+      "loss": 0.5782,
+      "step": 112
+    },
+    {
+      "epoch": 0.34649290916059794,
+      "grad_norm": 1.378460168838501,
+      "learning_rate": 9.994658191675141e-06,
+      "loss": 0.5653,
+      "step": 113
+    },
+    {
+      "epoch": 0.3495592180912227,
+      "grad_norm": 1.349794864654541,
+      "learning_rate": 9.994428505154362e-06,
+      "loss": 0.5672,
+      "step": 114
+    },
+    {
+      "epoch": 0.35262552702184746,
+      "grad_norm": 1.2933546304702759,
+      "learning_rate": 9.994193986758619e-06,
+      "loss": 0.6698,
+      "step": 115
+    },
+    {
+      "epoch": 0.3556918359524722,
+      "grad_norm": 1.2966620922088623,
+      "learning_rate": 9.993954636714796e-06,
+      "loss": 0.6409,
+      "step": 116
+    },
+    {
+      "epoch": 0.358758144883097,
+      "grad_norm": 1.6862447261810303,
+      "learning_rate": 9.993710455254454e-06,
+      "loss": 0.6606,
+      "step": 117
+    },
+    {
+      "epoch": 0.36182445381372175,
+      "grad_norm": 1.3929141759872437,
+      "learning_rate": 9.99346144261383e-06,
+      "loss": 0.599,
+      "step": 118
+    },
+    {
+      "epoch": 0.3648907627443465,
+      "grad_norm": 1.6301984786987305,
+      "learning_rate": 9.993207599033827e-06,
+      "loss": 0.649,
+      "step": 119
+    },
+    {
+      "epoch": 0.36795707167497127,
+      "grad_norm": 1.5956659317016602,
+      "learning_rate": 9.99294892476003e-06,
+      "loss": 0.6215,
+      "step": 120
+    },
+    {
+      "epoch": 0.37102338060559603,
+      "grad_norm": 1.3916531801223755,
+      "learning_rate": 9.992685420042694e-06,
+      "loss": 0.6626,
+      "step": 121
+    },
+    {
+      "epoch": 0.3740896895362208,
+      "grad_norm": 1.2992644309997559,
+      "learning_rate": 9.992417085136744e-06,
+      "loss": 0.5428,
+      "step": 122
+    },
+    {
+      "epoch": 0.37715599846684555,
+      "grad_norm": 1.2271054983139038,
+      "learning_rate": 9.992143920301788e-06,
+      "loss": 0.608,
+      "step": 123
+    },
+    {
+      "epoch": 0.3802223073974703,
+      "grad_norm": 1.2169947624206543,
+      "learning_rate": 9.991865925802096e-06,
+      "loss": 0.5608,
+      "step": 124
+    },
+    {
+      "epoch": 0.3832886163280951,
+      "grad_norm": 1.423193097114563,
+      "learning_rate": 9.991583101906613e-06,
+      "loss": 0.6193,
+      "step": 125
+    },
+    {
+      "epoch": 0.38635492525871984,
+      "grad_norm": 1.1559628248214722,
+      "learning_rate": 9.991295448888961e-06,
+      "loss": 0.5428,
+      "step": 126
+    },
+    {
+      "epoch": 0.3894212341893446,
+      "grad_norm": 1.1540210247039795,
+      "learning_rate": 9.99100296702743e-06,
+      "loss": 0.5916,
+      "step": 127
+    },
+    {
+      "epoch": 0.39248754311996936,
+      "grad_norm": 1.3560632467269897,
+      "learning_rate": 9.990705656604982e-06,
+      "loss": 0.5624,
+      "step": 128
+    },
+    {
+      "epoch": 0.3955538520505941,
+      "grad_norm": 1.2158130407333374,
+      "learning_rate": 9.990403517909251e-06,
+      "loss": 0.5982,
+      "step": 129
+    },
+    {
+      "epoch": 0.3986201609812189,
+      "grad_norm": 1.254124402999878,
+      "learning_rate": 9.99009655123254e-06,
+      "loss": 0.5335,
+      "step": 130
+    },
+    {
+      "epoch": 0.4016864699118436,
+      "grad_norm": 2.5791096687316895,
+      "learning_rate": 9.98978475687183e-06,
+      "loss": 0.6424,
+      "step": 131
+    },
+    {
+      "epoch": 0.40475277884246835,
+      "grad_norm": 1.6048130989074707,
+      "learning_rate": 9.98946813512876e-06,
+      "loss": 0.6515,
+      "step": 132
+    },
+    {
+      "epoch": 0.4078190877730931,
+      "grad_norm": 1.490658164024353,
+      "learning_rate": 9.989146686309657e-06,
+      "loss": 0.5155,
+      "step": 133
+    },
+    {
+      "epoch": 0.4108853967037179,
+      "grad_norm": 1.3266921043395996,
+      "learning_rate": 9.988820410725497e-06,
+      "loss": 0.5607,
+      "step": 134
+    },
+    {
+      "epoch": 0.41395170563434264,
+      "grad_norm": 1.1469498872756958,
+      "learning_rate": 9.988489308691943e-06,
+      "loss": 0.6289,
+      "step": 135
+    },
+    {
+      "epoch": 0.4170180145649674,
+      "grad_norm": 1.5313925743103027,
+      "learning_rate": 9.988153380529316e-06,
+      "loss": 0.5974,
+      "step": 136
+    },
+    {
+      "epoch": 0.42008432349559216,
+      "grad_norm": 1.289024829864502,
+      "learning_rate": 9.987812626562615e-06,
+      "loss": 0.5788,
+      "step": 137
+    },
+    {
+      "epoch": 0.4231506324262169,
+      "grad_norm": 1.3981531858444214,
+      "learning_rate": 9.987467047121502e-06,
+      "loss": 0.6242,
+      "step": 138
+    },
+    {
+      "epoch": 0.4262169413568417,
+      "grad_norm": 1.3675987720489502,
+      "learning_rate": 9.987116642540307e-06,
+      "loss": 0.5827,
+      "step": 139
+    },
+    {
+      "epoch": 0.42928325028746644,
+      "grad_norm": 1.4098269939422607,
+      "learning_rate": 9.986761413158032e-06,
+      "loss": 0.5186,
+      "step": 140
+    },
+    {
+      "epoch": 0.4323495592180912,
+      "grad_norm": 1.328583836555481,
+      "learning_rate": 9.986401359318345e-06,
+      "loss": 0.6279,
+      "step": 141
+    },
+    {
+      "epoch": 0.43541586814871597,
+      "grad_norm": 1.379964828491211,
+      "learning_rate": 9.986036481369579e-06,
+      "loss": 0.6128,
+      "step": 142
+    },
+    {
+      "epoch": 0.43848217707934073,
+      "grad_norm": 1.404513955116272,
+      "learning_rate": 9.98566677966474e-06,
+      "loss": 0.6203,
+      "step": 143
+    },
+    {
+      "epoch": 0.4415484860099655,
+      "grad_norm": 1.285415768623352,
+      "learning_rate": 9.985292254561492e-06,
+      "loss": 0.5542,
+      "step": 144
+    },
+    {
+      "epoch": 0.44461479494059025,
+      "grad_norm": 1.604349136352539,
+      "learning_rate": 9.984912906422176e-06,
+      "loss": 0.5365,
+      "step": 145
+    },
+    {
+      "epoch": 0.447681103871215,
+      "grad_norm": 1.150905966758728,
+      "learning_rate": 9.984528735613788e-06,
+      "loss": 0.5441,
+      "step": 146
+    },
+    {
+      "epoch": 0.4507474128018398,
+      "grad_norm": 1.086539626121521,
+      "learning_rate": 9.984139742507999e-06,
+      "loss": 0.5014,
+      "step": 147
+    },
+    {
+      "epoch": 0.45381372173246454,
+      "grad_norm": 1.3198760747909546,
+      "learning_rate": 9.98374592748114e-06,
+      "loss": 0.557,
+      "step": 148
+    },
+    {
+      "epoch": 0.4568800306630893,
+      "grad_norm": 1.2699756622314453,
+      "learning_rate": 9.98334729091421e-06,
+      "loss": 0.6461,
+      "step": 149
+    },
+    {
+      "epoch": 0.45994633959371406,
+      "grad_norm": 1.2899307012557983,
+      "learning_rate": 9.98294383319287e-06,
+      "loss": 0.6349,
+      "step": 150
+    },
+    {
+      "epoch": 0.4630126485243388,
+      "grad_norm": 1.4127708673477173,
+      "learning_rate": 9.982535554707446e-06,
+      "loss": 0.7009,
+      "step": 151
+    },
+    {
+      "epoch": 0.4660789574549636,
+      "grad_norm": 1.1540488004684448,
+      "learning_rate": 9.982122455852928e-06,
+      "loss": 0.5746,
+      "step": 152
+    },
+    {
+      "epoch": 0.46914526638558834,
+      "grad_norm": 1.3686162233352661,
+      "learning_rate": 9.98170453702897e-06,
+      "loss": 0.5412,
+      "step": 153
+    },
+    {
+      "epoch": 0.4722115753162131,
+      "grad_norm": 1.168924331665039,
+      "learning_rate": 9.981281798639892e-06,
+      "loss": 0.5176,
+      "step": 154
+    },
+    {
+      "epoch": 0.47527788424683787,
+      "grad_norm": 1.4302713871002197,
+      "learning_rate": 9.980854241094671e-06,
+      "loss": 0.6231,
+      "step": 155
+    },
+    {
+      "epoch": 0.47834419317746263,
+      "grad_norm": 1.156389832496643,
+      "learning_rate": 9.980421864806947e-06,
+      "loss": 0.6283,
+      "step": 156
+    },
+    {
+      "epoch": 0.4814105021080874,
+      "grad_norm": 1.2555568218231201,
+      "learning_rate": 9.979984670195025e-06,
+      "loss": 0.5441,
+      "step": 157
+    },
+    {
+      "epoch": 0.48447681103871215,
+      "grad_norm": 1.1491152048110962,
+      "learning_rate": 9.97954265768187e-06,
+      "loss": 0.5562,
+      "step": 158
+    },
+    {
+      "epoch": 0.4875431199693369,
+      "grad_norm": 1.7037622928619385,
+      "learning_rate": 9.979095827695113e-06,
+      "loss": 0.5963,
+      "step": 159
+    },
+    {
+      "epoch": 0.4906094288999617,
+      "grad_norm": 1.5287169218063354,
+      "learning_rate": 9.978644180667034e-06,
+      "loss": 0.5817,
+      "step": 160
+    },
+    {
+      "epoch": 0.49367573783058644,
+      "grad_norm": 1.5015875101089478,
+      "learning_rate": 9.978187717034584e-06,
+      "loss": 0.589,
+      "step": 161
+    },
+    {
+      "epoch": 0.4967420467612112,
+      "grad_norm": 1.218470811843872,
+      "learning_rate": 9.977726437239368e-06,
+      "loss": 0.6042,
+      "step": 162
+    },
+    {
+      "epoch": 0.49980835569183596,
+      "grad_norm": 1.4837976694107056,
+      "learning_rate": 9.977260341727657e-06,
+      "loss": 0.5706,
+      "step": 163
+    },
+    {
+      "epoch": 0.5028746646224607,
+      "grad_norm": 1.1982754468917847,
+      "learning_rate": 9.976789430950371e-06,
+      "loss": 0.6037,
+      "step": 164
+    },
+    {
+      "epoch": 0.5059409735530854,
+      "grad_norm": 1.3165373802185059,
+      "learning_rate": 9.976313705363099e-06,
+      "loss": 0.6074,
+      "step": 165
+    },
+    {
+      "epoch": 0.5090072824837102,
+      "grad_norm": 1.418428659439087,
+      "learning_rate": 9.975833165426079e-06,
+      "loss": 0.6106,
+      "step": 166
+    },
+    {
+      "epoch": 0.512073591414335,
+      "grad_norm": 1.275534987449646,
+      "learning_rate": 9.975347811604213e-06,
+      "loss": 0.6015,
+      "step": 167
+    },
+    {
+      "epoch": 0.5151399003449597,
+      "grad_norm": 1.3691564798355103,
+      "learning_rate": 9.974857644367056e-06,
+      "loss": 0.5853,
+      "step": 168
+    },
+    {
+      "epoch": 0.5182062092755845,
+      "grad_norm": 1.290286898612976,
+      "learning_rate": 9.974362664188825e-06,
+      "loss": 0.606,
+      "step": 169
+    },
+    {
+      "epoch": 0.5212725182062092,
+      "grad_norm": 1.3604097366333008,
+      "learning_rate": 9.973862871548389e-06,
+      "loss": 0.5576,
+      "step": 170
+    },
+    {
+      "epoch": 0.524338827136834,
+      "grad_norm": 1.1056456565856934,
+      "learning_rate": 9.973358266929272e-06,
+      "loss": 0.5625,
+      "step": 171
+    },
+    {
+      "epoch": 0.5274051360674588,
+      "grad_norm": 1.2385663986206055,
+      "learning_rate": 9.972848850819657e-06,
+      "loss": 0.5222,
+      "step": 172
+    },
+    {
+      "epoch": 0.5304714449980835,
+      "grad_norm": 1.1576796770095825,
+      "learning_rate": 9.972334623712381e-06,
+      "loss": 0.5891,
+      "step": 173
+    },
+    {
+      "epoch": 0.5335377539287083,
+      "grad_norm": 1.6581560373306274,
+      "learning_rate": 9.971815586104932e-06,
+      "loss": 0.5549,
+      "step": 174
+    },
+    {
+      "epoch": 0.536604062859333,
+      "grad_norm": 1.699034571647644,
+      "learning_rate": 9.971291738499456e-06,
+      "loss": 0.5884,
+      "step": 175
+    },
+    {
+      "epoch": 0.5396703717899578,
+      "grad_norm": 1.884663462638855,
+      "learning_rate": 9.97076308140275e-06,
+      "loss": 0.5892,
+      "step": 176
+    },
+    {
+      "epoch": 0.5427366807205826,
+      "grad_norm": 1.313745379447937,
+      "learning_rate": 9.970229615326267e-06,
+      "loss": 0.5954,
+      "step": 177
+    },
+    {
+      "epoch": 0.5458029896512073,
+      "grad_norm": 1.38603675365448,
+      "learning_rate": 9.96969134078611e-06,
+      "loss": 0.6078,
+      "step": 178
+    },
+    {
+      "epoch": 0.5488692985818321,
+      "grad_norm": 1.3190451860427856,
+      "learning_rate": 9.96914825830303e-06,
+      "loss": 0.594,
+      "step": 179
+    },
+    {
+      "epoch": 0.5519356075124568,
+      "grad_norm": 1.2696350812911987,
+      "learning_rate": 9.968600368402438e-06,
+      "loss": 0.5568,
+      "step": 180
+    },
+    {
+      "epoch": 0.5550019164430816,
+      "grad_norm": 1.13862943649292,
+      "learning_rate": 9.968047671614394e-06,
+      "loss": 0.5678,
+      "step": 181
+    },
+    {
+      "epoch": 0.5580682253737064,
+      "grad_norm": 1.2977395057678223,
+      "learning_rate": 9.967490168473601e-06,
+      "loss": 0.6177,
+      "step": 182
+    },
+    {
+      "epoch": 0.5611345343043311,
+      "grad_norm": 1.515966534614563,
+      "learning_rate": 9.966927859519419e-06,
+      "loss": 0.5937,
+      "step": 183
+    },
+    {
+      "epoch": 0.5642008432349559,
+      "grad_norm": 1.3620631694793701,
+      "learning_rate": 9.966360745295856e-06,
+      "loss": 0.6077,
+      "step": 184
+    },
+    {
+      "epoch": 0.5672671521655807,
+      "grad_norm": 1.2670691013336182,
+      "learning_rate": 9.965788826351568e-06,
+      "loss": 0.5681,
+      "step": 185
+    },
+    {
+      "epoch": 0.5703334610962054,
+      "grad_norm": 1.2297751903533936,
+      "learning_rate": 9.96521210323986e-06,
+      "loss": 0.5527,
+      "step": 186
+    },
+    {
+      "epoch": 0.5733997700268302,
+      "grad_norm": 1.1727638244628906,
+      "learning_rate": 9.964630576518684e-06,
+      "loss": 0.5439,
+      "step": 187
+    },
+    {
+      "epoch": 0.5764660789574549,
+      "grad_norm": 1.306315541267395,
+      "learning_rate": 9.96404424675064e-06,
+      "loss": 0.6183,
+      "step": 188
+    },
+    {
+      "epoch": 0.5795323878880797,
+      "grad_norm": 1.2800726890563965,
+      "learning_rate": 9.963453114502976e-06,
+      "loss": 0.5831,
+      "step": 189
+    },
+    {
+      "epoch": 0.5825986968187045,
+      "grad_norm": 1.1831467151641846,
+      "learning_rate": 9.962857180347582e-06,
+      "loss": 0.521,
+      "step": 190
+    },
+    {
+      "epoch": 0.5856650057493292,
+      "grad_norm": 1.3062933683395386,
+      "learning_rate": 9.962256444860997e-06,
+      "loss": 0.5684,
+      "step": 191
+    },
+    {
+      "epoch": 0.588731314679954,
+      "grad_norm": 1.2360109090805054,
+      "learning_rate": 9.961650908624406e-06,
+      "loss": 0.5676,
+      "step": 192
+    },
+    {
+      "epoch": 0.5917976236105787,
+      "grad_norm": 1.310551643371582,
+      "learning_rate": 9.961040572223636e-06,
+      "loss": 0.5632,
+      "step": 193
+    },
+    {
+      "epoch": 0.5948639325412035,
+      "grad_norm": 1.20192551612854,
+      "learning_rate": 9.960425436249156e-06,
+      "loss": 0.5141,
+      "step": 194
+    },
+    {
+      "epoch": 0.5979302414718283,
+      "grad_norm": 1.1117688417434692,
+      "learning_rate": 9.959805501296087e-06,
+      "loss": 0.6622,
+      "step": 195
+    },
+    {
+      "epoch": 0.600996550402453,
+      "grad_norm": 1.124586820602417,
+      "learning_rate": 9.959180767964183e-06,
+      "loss": 0.5676,
+      "step": 196
+    },
+    {
+      "epoch": 0.6040628593330778,
+      "grad_norm": 1.136702299118042,
+      "learning_rate": 9.958551236857843e-06,
+      "loss": 0.5238,
+      "step": 197
+    },
+    {
+      "epoch": 0.6071291682637026,
+      "grad_norm": 1.1443421840667725,
+      "learning_rate": 9.957916908586112e-06,
+      "loss": 0.5825,
+      "step": 198
+    },
+    {
+      "epoch": 0.6101954771943273,
+      "grad_norm": 1.3255038261413574,
+      "learning_rate": 9.957277783762672e-06,
+      "loss": 0.6222,
+      "step": 199
+    },
+    {
+      "epoch": 0.6132617861249521,
+      "grad_norm": 1.40571928024292,
+      "learning_rate": 9.956633863005845e-06,
+      "loss": 0.6709,
+      "step": 200
+    },
+    {
+      "epoch": 0.6163280950555768,
+      "grad_norm": 1.2186510562896729,
+      "learning_rate": 9.955985146938595e-06,
+      "loss": 0.5416,
+      "step": 201
+    },
+    {
+      "epoch": 0.6193944039862016,
+      "grad_norm": 1.4977316856384277,
+      "learning_rate": 9.955331636188523e-06,
+      "loss": 0.5715,
+      "step": 202
+    },
+    {
+      "epoch": 0.6224607129168264,
+      "grad_norm": 1.1389449834823608,
+      "learning_rate": 9.954673331387872e-06,
+      "loss": 0.5625,
+      "step": 203
+    },
+    {
+      "epoch": 0.6255270218474511,
+      "grad_norm": 1.199899673461914,
+      "learning_rate": 9.95401023317352e-06,
+      "loss": 0.5577,
+      "step": 204
+    },
+    {
+      "epoch": 0.6285933307780759,
+      "grad_norm": 1.3944934606552124,
+      "learning_rate": 9.953342342186984e-06,
+      "loss": 0.5359,
+      "step": 205
+    },
+    {
+      "epoch": 0.6316596397087006,
+      "grad_norm": 1.3514692783355713,
+      "learning_rate": 9.952669659074416e-06,
+      "loss": 0.5419,
+      "step": 206
+    },
+    {
+      "epoch": 0.6347259486393254,
+      "grad_norm": 1.257280945777893,
+      "learning_rate": 9.951992184486605e-06,
+      "loss": 0.6755,
+      "step": 207
+    },
+    {
+      "epoch": 0.6377922575699502,
+      "grad_norm": 1.2243086099624634,
+      "learning_rate": 9.951309919078978e-06,
+      "loss": 0.6197,
+      "step": 208
+    },
+    {
+      "epoch": 0.6408585665005749,
+      "grad_norm": 1.4111242294311523,
+      "learning_rate": 9.950622863511591e-06,
+      "loss": 0.5863,
+      "step": 209
+    },
+    {
+      "epoch": 0.6439248754311997,
+      "grad_norm": 1.4944766759872437,
+      "learning_rate": 9.949931018449144e-06,
+      "loss": 0.6013,
+      "step": 210
+    },
+    {
+      "epoch": 0.6469911843618245,
+      "grad_norm": 1.221962332725525,
+      "learning_rate": 9.949234384560958e-06,
+      "loss": 0.5949,
+      "step": 211
+    },
+    {
+      "epoch": 0.6500574932924492,
+      "grad_norm": 1.2524234056472778,
+      "learning_rate": 9.948532962520998e-06,
+      "loss": 0.5804,
+      "step": 212
+    },
+    {
+      "epoch": 0.653123802223074,
+      "grad_norm": 1.4645860195159912,
+      "learning_rate": 9.947826753007856e-06,
+      "loss": 0.5906,
+      "step": 213
+    },
+    {
+      "epoch": 0.6561901111536987,
+      "grad_norm": 1.3019660711288452,
+      "learning_rate": 9.947115756704755e-06,
+      "loss": 0.58,
+      "step": 214
+    },
+    {
+      "epoch": 0.6592564200843235,
+      "grad_norm": 1.1700494289398193,
+      "learning_rate": 9.946399974299552e-06,
+      "loss": 0.4977,
+      "step": 215
+    },
+    {
+      "epoch": 0.6623227290149483,
+      "grad_norm": 1.2081421613693237,
+      "learning_rate": 9.945679406484731e-06,
+      "loss": 0.657,
+      "step": 216
+    },
+    {
+      "epoch": 0.665389037945573,
+      "grad_norm": 1.139465570449829,
+      "learning_rate": 9.94495405395741e-06,
+      "loss": 0.5609,
+      "step": 217
+    },
+    {
+      "epoch": 0.6684553468761978,
+      "grad_norm": 1.2909924983978271,
+      "learning_rate": 9.944223917419332e-06,
+      "loss": 0.5386,
+      "step": 218
+    },
+    {
+      "epoch": 0.6715216558068225,
+      "grad_norm": 1.2462562322616577,
+      "learning_rate": 9.943488997576871e-06,
+      "loss": 0.5485,
+      "step": 219
+    },
+    {
+      "epoch": 0.6745879647374473,
+      "grad_norm": 1.097688913345337,
+      "learning_rate": 9.942749295141025e-06,
+      "loss": 0.54,
+      "step": 220
+    },
+    {
+      "epoch": 0.6776542736680721,
+      "grad_norm": 1.1347556114196777,
+      "learning_rate": 9.942004810827423e-06,
+      "loss": 0.5637,
+      "step": 221
+    },
+    {
+      "epoch": 0.6807205825986968,
+      "grad_norm": 1.1871147155761719,
+      "learning_rate": 9.941255545356321e-06,
+      "loss": 0.6179,
+      "step": 222
+    },
+    {
+      "epoch": 0.6837868915293216,
+      "grad_norm": 1.2758738994598389,
+      "learning_rate": 9.940501499452593e-06,
+      "loss": 0.5582,
+      "step": 223
+    },
+    {
+      "epoch": 0.6868532004599464,
+      "grad_norm": 1.298793911933899,
+      "learning_rate": 9.939742673845747e-06,
+      "loss": 0.5128,
+      "step": 224
+    },
+    {
+      "epoch": 0.6899195093905711,
+      "grad_norm": 1.9454792737960815,
+      "learning_rate": 9.938979069269909e-06,
+      "loss": 0.5316,
+      "step": 225
+    },
+    {
+      "epoch": 0.6929858183211959,
+      "grad_norm": 1.4250558614730835,
+      "learning_rate": 9.938210686463834e-06,
+      "loss": 0.5998,
+      "step": 226
+    },
+    {
+      "epoch": 0.6960521272518206,
+      "grad_norm": 1.9154913425445557,
+      "learning_rate": 9.93743752617089e-06,
+      "loss": 0.5922,
+      "step": 227
+    },
+    {
+      "epoch": 0.6991184361824454,
+      "grad_norm": 1.552946925163269,
+      "learning_rate": 9.936659589139078e-06,
+      "loss": 0.61,
+      "step": 228
+    },
+    {
+      "epoch": 0.7021847451130702,
+      "grad_norm": 1.3364359140396118,
+      "learning_rate": 9.935876876121016e-06,
+      "loss": 0.6085,
+      "step": 229
+    },
+    {
+      "epoch": 0.7052510540436949,
+      "grad_norm": 1.5024923086166382,
+      "learning_rate": 9.935089387873938e-06,
+      "loss": 0.5599,
+      "step": 230
+    },
+    {
+      "epoch": 0.7083173629743197,
+      "grad_norm": 1.3065555095672607,
+      "learning_rate": 9.934297125159707e-06,
+      "loss": 0.6381,
+      "step": 231
+    },
+    {
+      "epoch": 0.7113836719049444,
+      "grad_norm": 1.158210039138794,
+      "learning_rate": 9.933500088744795e-06,
+      "loss": 0.6796,
+      "step": 232
+    },
+    {
+      "epoch": 0.7144499808355692,
+      "grad_norm": 1.2026755809783936,
+      "learning_rate": 9.932698279400301e-06,
+      "loss": 0.5747,
+      "step": 233
+    },
+    {
+      "epoch": 0.717516289766194,
+      "grad_norm": 1.2283495664596558,
+      "learning_rate": 9.931891697901937e-06,
+      "loss": 0.5401,
+      "step": 234
+    },
+    {
+      "epoch": 0.7205825986968187,
+      "grad_norm": 1.4405039548873901,
+      "learning_rate": 9.93108034503003e-06,
+      "loss": 0.5837,
+      "step": 235
+    },
+    {
+      "epoch": 0.7236489076274435,
+      "grad_norm": 1.0767070055007935,
+      "learning_rate": 9.93026422156953e-06,
+      "loss": 0.5265,
+      "step": 236
+    },
+    {
+      "epoch": 0.7267152165580683,
+      "grad_norm": 1.2622807025909424,
+      "learning_rate": 9.929443328309995e-06,
+      "loss": 0.5767,
+      "step": 237
+    },
+    {
+      "epoch": 0.729781525488693,
+      "grad_norm": 1.8162665367126465,
+      "learning_rate": 9.928617666045603e-06,
+      "loss": 0.6198,
+      "step": 238
+    },
+    {
+      "epoch": 0.7328478344193178,
+      "grad_norm": 1.1307610273361206,
+      "learning_rate": 9.927787235575141e-06,
+      "loss": 0.5548,
+      "step": 239
+    },
+    {
+      "epoch": 0.7359141433499425,
+      "grad_norm": 1.3943837881088257,
+      "learning_rate": 9.926952037702013e-06,
+      "loss": 0.6017,
+      "step": 240
+    },
+    {
+      "epoch": 0.7389804522805673,
+      "grad_norm": 1.1399344205856323,
+      "learning_rate": 9.926112073234236e-06,
+      "loss": 0.5848,
+      "step": 241
+    },
+    {
+      "epoch": 0.7420467612111921,
+      "grad_norm": 1.063624620437622,
+      "learning_rate": 9.925267342984431e-06,
+      "loss": 0.5012,
+      "step": 242
+    },
+    {
+      "epoch": 0.7451130701418168,
+      "grad_norm": 1.1229875087738037,
+      "learning_rate": 9.924417847769836e-06,
+      "loss": 0.5875,
+      "step": 243
+    },
+    {
+      "epoch": 0.7481793790724416,
+      "grad_norm": 1.106398105621338,
+      "learning_rate": 9.923563588412302e-06,
+      "loss": 0.5628,
+      "step": 244
+    },
+    {
+      "epoch": 0.7512456880030663,
+      "grad_norm": 1.257165551185608,
+      "learning_rate": 9.922704565738281e-06,
+      "loss": 0.5492,
+      "step": 245
+    },
+    {
+      "epoch": 0.7543119969336911,
+      "grad_norm": 1.7508597373962402,
+      "learning_rate": 9.921840780578838e-06,
+      "loss": 0.5559,
+      "step": 246
+    },
+    {
+      "epoch": 0.7573783058643159,
+      "grad_norm": 1.2271780967712402,
+      "learning_rate": 9.920972233769644e-06,
+      "loss": 0.5749,
+      "step": 247
+    },
+    {
+      "epoch": 0.7604446147949406,
+      "grad_norm": 1.26109778881073,
+      "learning_rate": 9.92009892615098e-06,
+      "loss": 0.6323,
+      "step": 248
+    },
+    {
+      "epoch": 0.7635109237255654,
+      "grad_norm": 1.2538588047027588,
+      "learning_rate": 9.919220858567725e-06,
+      "loss": 0.5657,
+      "step": 249
+    },
+    {
+      "epoch": 0.7665772326561902,
+      "grad_norm": 1.2283259630203247,
+      "learning_rate": 9.918338031869373e-06,
+      "loss": 0.5895,
+      "step": 250
+    },
+    {
+      "epoch": 0.7696435415868149,
+      "grad_norm": 1.3043975830078125,
+      "learning_rate": 9.917450446910014e-06,
+      "loss": 0.6051,
+      "step": 251
+    },
+    {
+      "epoch": 0.7727098505174397,
+      "grad_norm": 1.1411467790603638,
+      "learning_rate": 9.916558104548346e-06,
+      "loss": 0.5874,
+      "step": 252
+    },
+    {
+      "epoch": 0.7757761594480644,
+      "grad_norm": 1.595882773399353,
+      "learning_rate": 9.91566100564767e-06,
+      "loss": 0.5436,
+      "step": 253
+    },
+    {
+      "epoch": 0.7788424683786892,
+      "grad_norm": 1.166404366493225,
+      "learning_rate": 9.914759151075885e-06,
+      "loss": 0.5632,
+      "step": 254
+    },
+    {
+      "epoch": 0.781908777309314,
+      "grad_norm": 1.1772657632827759,
+      "learning_rate": 9.913852541705493e-06,
+      "loss": 0.6405,
+      "step": 255
+    },
+    {
+      "epoch": 0.7849750862399387,
+      "grad_norm": 1.1861871480941772,
+      "learning_rate": 9.912941178413597e-06,
+      "loss": 0.6008,
+      "step": 256
+    },
+    {
+      "epoch": 0.7880413951705635,
+      "grad_norm": 1.3676292896270752,
+      "learning_rate": 9.912025062081899e-06,
+      "loss": 0.5217,
+      "step": 257
+    },
+    {
+      "epoch": 0.7911077041011882,
+      "grad_norm": 1.1941781044006348,
+      "learning_rate": 9.911104193596695e-06,
+      "loss": 0.5233,
+      "step": 258
+    },
+    {
+      "epoch": 0.794174013031813,
+      "grad_norm": 1.3091732263565063,
+      "learning_rate": 9.910178573848883e-06,
+      "loss": 0.6155,
+      "step": 259
+    },
+    {
+      "epoch": 0.7972403219624378,
+      "grad_norm": 1.139570951461792,
+      "learning_rate": 9.909248203733961e-06,
+      "loss": 0.5365,
+      "step": 260
+    },
+    {
+      "epoch": 0.8003066308930624,
+      "grad_norm": 1.2685056924819946,
+      "learning_rate": 9.908313084152012e-06,
+      "loss": 0.6398,
+      "step": 261
+    },
+    {
+      "epoch": 0.8033729398236872,
+      "grad_norm": 1.1302436590194702,
+      "learning_rate": 9.907373216007727e-06,
+      "loss": 0.5849,
+      "step": 262
+    },
+    {
+      "epoch": 0.8064392487543119,
+      "grad_norm": 1.07113778591156,
+      "learning_rate": 9.906428600210378e-06,
+      "loss": 0.6114,
+      "step": 263
+    },
+    {
+      "epoch": 0.8095055576849367,
+      "grad_norm": 1.2171744108200073,
+      "learning_rate": 9.905479237673841e-06,
+      "loss": 0.6419,
+      "step": 264
+    },
+    {
+      "epoch": 0.8125718666155615,
+      "grad_norm": 1.642828345298767,
+      "learning_rate": 9.904525129316576e-06,
+      "loss": 0.5992,
+      "step": 265
+    },
+    {
+      "epoch": 0.8156381755461862,
+      "grad_norm": 1.9515584707260132,
+      "learning_rate": 9.90356627606164e-06,
+      "loss": 0.5474,
+      "step": 266
+    },
+    {
+      "epoch": 0.818704484476811,
+      "grad_norm": 1.2620511054992676,
+      "learning_rate": 9.90260267883668e-06,
+      "loss": 0.625,
+      "step": 267
+    },
+    {
+      "epoch": 0.8217707934074358,
+      "grad_norm": 1.3877058029174805,
+      "learning_rate": 9.901634338573926e-06,
+      "loss": 0.6071,
+      "step": 268
+    },
+    {
+      "epoch": 0.8248371023380605,
+      "grad_norm": 1.811902403831482,
+      "learning_rate": 9.900661256210207e-06,
+      "loss": 0.5573,
+      "step": 269
+    },
+    {
+      "epoch": 0.8279034112686853,
+      "grad_norm": 1.1542803049087524,
+      "learning_rate": 9.899683432686933e-06,
+      "loss": 0.592,
+      "step": 270
+    },
+    {
+      "epoch": 0.83096972019931,
+      "grad_norm": 1.1733490228652954,
+      "learning_rate": 9.898700868950102e-06,
+      "loss": 0.5948,
+      "step": 271
+    },
+    {
+      "epoch": 0.8340360291299348,
+      "grad_norm": 1.0578864812850952,
+      "learning_rate": 9.897713565950297e-06,
+      "loss": 0.483,
+      "step": 272
+    },
+    {
+      "epoch": 0.8371023380605596,
+      "grad_norm": 1.1385244131088257,
+      "learning_rate": 9.896721524642689e-06,
+      "loss": 0.5959,
+      "step": 273
+    },
+    {
+      "epoch": 0.8401686469911843,
+      "grad_norm": 1.5948585271835327,
+      "learning_rate": 9.895724745987029e-06,
+      "loss": 0.6105,
+      "step": 274
+    },
+    {
+      "epoch": 0.8432349559218091,
+      "grad_norm": 1.1426746845245361,
+      "learning_rate": 9.894723230947658e-06,
+      "loss": 0.5761,
+      "step": 275
+    },
+    {
+      "epoch": 0.8463012648524338,
+      "grad_norm": 1.1893310546875,
+      "learning_rate": 9.893716980493492e-06,
+      "loss": 0.6312,
+      "step": 276
+    },
+    {
+      "epoch": 0.8493675737830586,
+      "grad_norm": 1.2711292505264282,
+      "learning_rate": 9.892705995598028e-06,
+      "loss": 0.5475,
+      "step": 277
+    },
+    {
+      "epoch": 0.8524338827136834,
+      "grad_norm": 1.291114330291748,
+      "learning_rate": 9.891690277239352e-06,
+      "loss": 0.5956,
+      "step": 278
+    },
+    {
+      "epoch": 0.8555001916443081,
+      "grad_norm": 1.2355763912200928,
+      "learning_rate": 9.890669826400121e-06,
+      "loss": 0.5848,
+      "step": 279
+    },
+    {
+      "epoch": 0.8585665005749329,
+      "grad_norm": 1.1467026472091675,
+      "learning_rate": 9.889644644067573e-06,
+      "loss": 0.5739,
+      "step": 280
+    },
+    {
+      "epoch": 0.8616328095055577,
+      "grad_norm": 1.2425200939178467,
+      "learning_rate": 9.888614731233523e-06,
+      "loss": 0.5706,
+      "step": 281
+    },
+    {
+      "epoch": 0.8646991184361824,
+      "grad_norm": 1.3056875467300415,
+      "learning_rate": 9.887580088894367e-06,
+      "loss": 0.5517,
+      "step": 282
+    },
+    {
+      "epoch": 0.8677654273668072,
+      "grad_norm": 1.1555942296981812,
+      "learning_rate": 9.88654071805107e-06,
+      "loss": 0.5779,
+      "step": 283
+    },
+    {
+      "epoch": 0.8708317362974319,
+      "grad_norm": 1.172336220741272,
+      "learning_rate": 9.885496619709173e-06,
+      "loss": 0.6184,
+      "step": 284
+    },
+    {
+      "epoch": 0.8738980452280567,
+      "grad_norm": 1.2276782989501953,
+      "learning_rate": 9.884447794878796e-06,
+      "loss": 0.6544,
+      "step": 285
+    },
+    {
+      "epoch": 0.8769643541586815,
+      "grad_norm": 1.0777740478515625,
+      "learning_rate": 9.883394244574624e-06,
+      "loss": 0.5711,
+      "step": 286
+    },
+    {
+      "epoch": 0.8800306630893062,
+      "grad_norm": 1.2766623497009277,
+      "learning_rate": 9.882335969815919e-06,
+      "loss": 0.5839,
+      "step": 287
+    },
+    {
+      "epoch": 0.883096972019931,
+      "grad_norm": 1.161967396736145,
+      "learning_rate": 9.881272971626515e-06,
+      "loss": 0.5959,
+      "step": 288
+    },
+    {
+      "epoch": 0.8861632809505557,
+      "grad_norm": 1.1648389101028442,
+      "learning_rate": 9.880205251034807e-06,
+      "loss": 0.5606,
+      "step": 289
+    },
+    {
+      "epoch": 0.8892295898811805,
+      "grad_norm": 1.2798397541046143,
+      "learning_rate": 9.87913280907377e-06,
+      "loss": 0.6505,
+      "step": 290
+    },
+    {
+      "epoch": 0.8922958988118053,
+      "grad_norm": 1.1248582601547241,
+      "learning_rate": 9.878055646780938e-06,
+      "loss": 0.5577,
+      "step": 291
+    },
+    {
+      "epoch": 0.89536220774243,
+      "grad_norm": 1.2309131622314453,
+      "learning_rate": 9.876973765198414e-06,
+      "loss": 0.6142,
+      "step": 292
+    },
+    {
+      "epoch": 0.8984285166730548,
+      "grad_norm": 1.280797004699707,
+      "learning_rate": 9.87588716537287e-06,
+      "loss": 0.6134,
+      "step": 293
+    },
+    {
+      "epoch": 0.9014948256036796,
+      "grad_norm": 1.0746029615402222,
+      "learning_rate": 9.87479584835554e-06,
+      "loss": 0.4581,
+      "step": 294
+    },
+    {
+      "epoch": 0.9045611345343043,
+      "grad_norm": 1.1132365465164185,
+      "learning_rate": 9.873699815202222e-06,
+      "loss": 0.5337,
+      "step": 295
+    },
+    {
+      "epoch": 0.9076274434649291,
+      "grad_norm": 1.0475718975067139,
+      "learning_rate": 9.872599066973276e-06,
+      "loss": 0.5762,
+      "step": 296
+    },
+    {
+      "epoch": 0.9106937523955538,
+      "grad_norm": 1.2082624435424805,
+      "learning_rate": 9.871493604733626e-06,
+      "loss": 0.5445,
+      "step": 297
+    },
+    {
+      "epoch": 0.9137600613261786,
+      "grad_norm": 1.1644026041030884,
+      "learning_rate": 9.87038342955275e-06,
+      "loss": 0.5242,
+      "step": 298
+    },
+    {
+      "epoch": 0.9168263702568034,
+      "grad_norm": 1.1655209064483643,
+      "learning_rate": 9.869268542504695e-06,
+      "loss": 0.5568,
+      "step": 299
+    },
+    {
+      "epoch": 0.9198926791874281,
+      "grad_norm": 1.3931217193603516,
+      "learning_rate": 9.868148944668061e-06,
+      "loss": 0.6037,
+      "step": 300
+    },
+    {
+      "epoch": 0.9229589881180529,
+      "grad_norm": 1.0738801956176758,
+      "learning_rate": 9.867024637126006e-06,
+      "loss": 0.5807,
+      "step": 301
+    },
+    {
+      "epoch": 0.9260252970486776,
+      "grad_norm": 1.2301298379898071,
+      "learning_rate": 9.865895620966241e-06,
+      "loss": 0.5418,
+      "step": 302
+    },
+    {
+      "epoch": 0.9290916059793024,
+      "grad_norm": 1.1297430992126465,
+      "learning_rate": 9.864761897281043e-06,
+      "loss": 0.5892,
+      "step": 303
+    },
+    {
+      "epoch": 0.9321579149099272,
+      "grad_norm": 1.1680104732513428,
+      "learning_rate": 9.863623467167232e-06,
+      "loss": 0.6066,
+      "step": 304
+    },
+    {
+      "epoch": 0.9352242238405519,
+      "grad_norm": 1.284200668334961,
+      "learning_rate": 9.862480331726186e-06,
+      "loss": 0.615,
+      "step": 305
+    },
+    {
+      "epoch": 0.9382905327711767,
+      "grad_norm": 1.1131356954574585,
+      "learning_rate": 9.861332492063835e-06,
+      "loss": 0.6144,
+      "step": 306
+    },
+    {
+      "epoch": 0.9413568417018015,
+      "grad_norm": 1.2503700256347656,
+      "learning_rate": 9.860179949290661e-06,
+      "loss": 0.5335,
+      "step": 307
+    },
+    {
+      "epoch": 0.9444231506324262,
+      "grad_norm": 1.1401102542877197,
+      "learning_rate": 9.859022704521695e-06,
+      "loss": 0.6011,
+      "step": 308
+    },
+    {
+      "epoch": 0.947489459563051,
+      "grad_norm": 1.0754965543746948,
+      "learning_rate": 9.857860758876516e-06,
+      "loss": 0.497,
+      "step": 309
+    },
+    {
+      "epoch": 0.9505557684936757,
+      "grad_norm": 1.179566502571106,
+      "learning_rate": 9.85669411347925e-06,
+      "loss": 0.5605,
+      "step": 310
+    },
+    {
+      "epoch": 0.9536220774243005,
+      "grad_norm": 1.3431735038757324,
+      "learning_rate": 9.855522769458576e-06,
+      "loss": 0.5284,
+      "step": 311
+    },
+    {
+      "epoch": 0.9566883863549253,
+      "grad_norm": 1.2126455307006836,
+      "learning_rate": 9.854346727947712e-06,
+      "loss": 0.6169,
+      "step": 312
+    },
+    {
+      "epoch": 0.95975469528555,
+      "grad_norm": 1.1429709196090698,
+      "learning_rate": 9.853165990084422e-06,
+      "loss": 0.6198,
+      "step": 313
+    },
+    {
+      "epoch": 0.9628210042161748,
+      "grad_norm": 1.3174223899841309,
+      "learning_rate": 9.851980557011016e-06,
+      "loss": 0.5923,
+      "step": 314
+    },
+    {
+      "epoch": 0.9658873131467995,
+      "grad_norm": 1.2224098443984985,
+      "learning_rate": 9.850790429874344e-06,
+      "loss": 0.6292,
+      "step": 315
+    },
+    {
+      "epoch": 0.9689536220774243,
+      "grad_norm": 1.163927674293518,
+      "learning_rate": 9.849595609825798e-06,
+      "loss": 0.6248,
+      "step": 316
+    },
+    {
+      "epoch": 0.9720199310080491,
+      "grad_norm": 1.374999761581421,
+      "learning_rate": 9.848396098021312e-06,
+      "loss": 0.5672,
+      "step": 317
+    },
+    {
+      "epoch": 0.9750862399386738,
+      "grad_norm": 1.2106223106384277,
+      "learning_rate": 9.847191895621354e-06,
+      "loss": 0.5657,
+      "step": 318
+    },
+    {
+      "epoch": 0.9781525488692986,
+      "grad_norm": 1.2192652225494385,
+      "learning_rate": 9.845983003790935e-06,
+      "loss": 0.6212,
+      "step": 319
+    },
+    {
+      "epoch": 0.9812188577999233,
+      "grad_norm": 1.2548232078552246,
+      "learning_rate": 9.844769423699603e-06,
+      "loss": 0.5447,
+      "step": 320
+    },
+    {
+      "epoch": 0.9842851667305481,
+      "grad_norm": 1.1989444494247437,
+      "learning_rate": 9.843551156521437e-06,
+      "loss": 0.6398,
+      "step": 321
+    },
+    {
+      "epoch": 0.9873514756611729,
+      "grad_norm": 1.3096071481704712,
+      "learning_rate": 9.842328203435054e-06,
+      "loss": 0.5516,
+      "step": 322
+    },
+    {
+      "epoch": 0.9904177845917976,
+      "grad_norm": 1.4656528234481812,
+      "learning_rate": 9.841100565623604e-06,
+      "loss": 0.5672,
+      "step": 323
+    },
+    {
+      "epoch": 0.9934840935224224,
+      "grad_norm": 1.2001426219940186,
+      "learning_rate": 9.839868244274769e-06,
+      "loss": 0.5759,
+      "step": 324
+    },
+    {
+      "epoch": 0.9965504024530472,
+      "grad_norm": 1.2918856143951416,
+      "learning_rate": 9.838631240580761e-06,
+      "loss": 0.6332,
+      "step": 325
+    },
+    {
+      "epoch": 0.9996167113836719,
+      "grad_norm": 1.255189299583435,
+      "learning_rate": 9.837389555738325e-06,
+      "loss": 0.6304,
+      "step": 326
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 3260,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 163,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.002154556467184e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c46372ec89f40bb5dfc3c06f0bf400eac5662f4c1d0bdeb06fc016860a1596c0
+size 6776