Upload checkpoint-5000

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 checkpoint-10000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-15000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-5000/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-5000/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128263,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 156960
+}

checkpoint-5000/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128009,
+    128258
+  ],
+  "pad_token_id": 128263,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.57.1"
+}

checkpoint-5000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b6fc6814a102a02fe4995fe8c087dc0dacd3828380395f4416c712a611fa1e1
+size 4991160848

checkpoint-5000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d3046d9bb45cf49e1e7423c03680ab1046d98f7318ada9aeea469d1153dfb41
+size 1610725592

checkpoint-5000/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_parameters": 3300928512,
+    "total_size": 6601857024
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-5000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d1bb7014817b3e24d8c94bdeb5c580e0ca0e22a7a320d34bc57b45db3b725e2
+size 13203939671

checkpoint-5000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:943132c13824a574bef79ad48eabeac032a12d8c47dc4b841793cf93fa37bb46
+size 16389

checkpoint-5000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00eb8e106883d6b7dc7a572e4c4b3d698f9ccee1a7e6c69fba0f96a399194a16
+size 16389

checkpoint-5000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2a1518bf24b2fbd90acea636179b1a8284d573a8ab6a74c089f7f7741cf3930
+size 16389

checkpoint-5000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f2f5abab75e42752ffe01e73e66db34e6df653a939e1300e7a83ce4b9b40e1
+size 16389

checkpoint-5000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8aeff9d4b76e5b37f768dae8ff1bb649b2c271622667dc37e18e4f4bbf98108
+size 16389

checkpoint-5000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dfdb0df595120e129157a8e23ba9a7718c5ea5d8c548f0f862b863ca75bb20d
+size 16389

checkpoint-5000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbfe125091abff5675514fceb73668612f02074859e61153b16cbe494ff74abe
+size 16389

checkpoint-5000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f3920c8b457b9bd645b9f797f0da9dadcf5190b52894a7695ce4d522733fa69
+size 16389

checkpoint-5000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c9405c30870e0cd059f8727fc5d56457e73a1098b46a3627e3b674e4c03c80f
+size 1465

checkpoint-5000/special_tokens_map.json

@@ -0,0 +1,165 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<angry>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<appalled>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<chuckle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<cry>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<curious>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<disappointed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<excited>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<exhale>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<gasp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<giggle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<gulp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<laugh>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<laugh_harder>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<mischievous>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sarcastic>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<scream>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sigh>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<sing>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<snort>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<whisper>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<custom_token_7>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-5000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c5e5b1d89b7e3738e5a5a4f93c326d8f3292ea83f9c560b8dbb6d66fb851973
+size 22853258

checkpoint-5000/trainer_state.json

@@ -0,0 +1,749 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.6567807141081932,
+  "eval_steps": 5000,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0003313727114572115,
+      "grad_norm": 2.828125,
+      "learning_rate": 0.0,
+      "loss": 4.1855,
+      "step": 1
+    },
+    {
+      "epoch": 0.016568635572860577,
+      "grad_norm": 3.234375,
+      "learning_rate": 9.017298490982701e-07,
+      "loss": 4.4012,
+      "step": 50
+    },
+    {
+      "epoch": 0.03313727114572115,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.8218623481781377e-06,
+      "loss": 4.3839,
+      "step": 100
+    },
+    {
+      "epoch": 0.04970590671858172,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.741994847258005e-06,
+      "loss": 4.345,
+      "step": 150
+    },
+    {
+      "epoch": 0.0662745422914423,
+      "grad_norm": 2.171875,
+      "learning_rate": 3.662127346337873e-06,
+      "loss": 4.2973,
+      "step": 200
+    },
+    {
+      "epoch": 0.08284317786430287,
+      "grad_norm": 1.5234375,
+      "learning_rate": 4.582259845417741e-06,
+      "loss": 4.224,
+      "step": 250
+    },
+    {
+      "epoch": 0.09941181343716345,
+      "grad_norm": 1.0234375,
+      "learning_rate": 5.502392344497608e-06,
+      "loss": 4.144,
+      "step": 300
+    },
+    {
+      "epoch": 0.11598044901002402,
+      "grad_norm": 1.0390625,
+      "learning_rate": 6.422524843577475e-06,
+      "loss": 4.0929,
+      "step": 350
+    },
+    {
+      "epoch": 0.1325490845828846,
+      "grad_norm": 1.1796875,
+      "learning_rate": 7.342657342657343e-06,
+      "loss": 4.0516,
+      "step": 400
+    },
+    {
+      "epoch": 0.14911772015574518,
+      "grad_norm": 1.390625,
+      "learning_rate": 8.26278984173721e-06,
+      "loss": 4.025,
+      "step": 450
+    },
+    {
+      "epoch": 0.16568635572860574,
+      "grad_norm": 1.515625,
+      "learning_rate": 9.182922340817078e-06,
+      "loss": 4.0113,
+      "step": 500
+    },
+    {
+      "epoch": 0.18225499130146633,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.0103054839896946e-05,
+      "loss": 4.0015,
+      "step": 550
+    },
+    {
+      "epoch": 0.1988236268743269,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.1023187338976813e-05,
+      "loss": 3.9936,
+      "step": 600
+    },
+    {
+      "epoch": 0.21539226244718748,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.1943319838056682e-05,
+      "loss": 3.9778,
+      "step": 650
+    },
+    {
+      "epoch": 0.23196089802004805,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.2863452337136547e-05,
+      "loss": 3.971,
+      "step": 700
+    },
+    {
+      "epoch": 0.24852953359290864,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.3783584836216415e-05,
+      "loss": 3.9664,
+      "step": 750
+    },
+    {
+      "epoch": 0.2650981691657692,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.4703717335296282e-05,
+      "loss": 3.9561,
+      "step": 800
+    },
+    {
+      "epoch": 0.28166680473862976,
+      "grad_norm": 1.0859375,
+      "learning_rate": 1.562384983437615e-05,
+      "loss": 3.9459,
+      "step": 850
+    },
+    {
+      "epoch": 0.29823544031149035,
+      "grad_norm": 1.0,
+      "learning_rate": 1.6543982333456018e-05,
+      "loss": 3.9438,
+      "step": 900
+    },
+    {
+      "epoch": 0.31480407588435094,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.7464114832535886e-05,
+      "loss": 3.9386,
+      "step": 950
+    },
+    {
+      "epoch": 0.3313727114572115,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.8384247331615755e-05,
+      "loss": 3.9285,
+      "step": 1000
+    },
+    {
+      "epoch": 0.34794134703007207,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.930437983069562e-05,
+      "loss": 3.9211,
+      "step": 1050
+    },
+    {
+      "epoch": 0.36450998260293266,
+      "grad_norm": 1.0859375,
+      "learning_rate": 2.022451232977549e-05,
+      "loss": 3.9216,
+      "step": 1100
+    },
+    {
+      "epoch": 0.38107861817579325,
+      "grad_norm": 0.9921875,
+      "learning_rate": 2.1144644828855357e-05,
+      "loss": 3.9118,
+      "step": 1150
+    },
+    {
+      "epoch": 0.3976472537486538,
+      "grad_norm": 1.0078125,
+      "learning_rate": 2.2064777327935222e-05,
+      "loss": 3.9059,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4142158893215144,
+      "grad_norm": 1.03125,
+      "learning_rate": 2.298490982701509e-05,
+      "loss": 3.9019,
+      "step": 1250
+    },
+    {
+      "epoch": 0.43078452489437496,
+      "grad_norm": 1.0234375,
+      "learning_rate": 2.390504232609496e-05,
+      "loss": 3.9035,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4473531604672355,
+      "grad_norm": 0.8671875,
+      "learning_rate": 2.4825174825174828e-05,
+      "loss": 3.8937,
+      "step": 1350
+    },
+    {
+      "epoch": 0.4639217960400961,
+      "grad_norm": 0.89453125,
+      "learning_rate": 2.5745307324254693e-05,
+      "loss": 3.881,
+      "step": 1400
+    },
+    {
+      "epoch": 0.4804904316129567,
+      "grad_norm": 0.90234375,
+      "learning_rate": 2.666543982333456e-05,
+      "loss": 3.8808,
+      "step": 1450
+    },
+    {
+      "epoch": 0.49705906718581727,
+      "grad_norm": 0.84375,
+      "learning_rate": 2.7585572322414427e-05,
+      "loss": 3.8782,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5136277027586779,
+      "grad_norm": 1.0,
+      "learning_rate": 2.8505704821494296e-05,
+      "loss": 3.8817,
+      "step": 1550
+    },
+    {
+      "epoch": 0.5301963383315385,
+      "grad_norm": 0.94140625,
+      "learning_rate": 2.942583732057416e-05,
+      "loss": 3.8718,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5467649739043989,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.034596981965403e-05,
+      "loss": 3.8675,
+      "step": 1650
+    },
+    {
+      "epoch": 0.5633336094772595,
+      "grad_norm": 0.75390625,
+      "learning_rate": 3.12661023187339e-05,
+      "loss": 3.8699,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5799022450501201,
+      "grad_norm": 0.77734375,
+      "learning_rate": 3.2186234817813766e-05,
+      "loss": 3.8645,
+      "step": 1750
+    },
+    {
+      "epoch": 0.5964708806229807,
+      "grad_norm": 0.86328125,
+      "learning_rate": 3.3106367316893635e-05,
+      "loss": 3.8601,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6130395161958413,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.4026499815973504e-05,
+      "loss": 3.8513,
+      "step": 1850
+    },
+    {
+      "epoch": 0.6296081517687019,
+      "grad_norm": 0.828125,
+      "learning_rate": 3.4946632315053365e-05,
+      "loss": 3.8583,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6461767873415625,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.5866764814133234e-05,
+      "loss": 3.857,
+      "step": 1950
+    },
+    {
+      "epoch": 0.662745422914423,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.67868973132131e-05,
+      "loss": 3.8488,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6793140584872835,
+      "grad_norm": 0.7734375,
+      "learning_rate": 3.770702981229297e-05,
+      "loss": 3.843,
+      "step": 2050
+    },
+    {
+      "epoch": 0.6958826940601441,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.862716231137284e-05,
+      "loss": 3.8413,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7124513296330047,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.954729481045271e-05,
+      "loss": 3.836,
+      "step": 2150
+    },
+    {
+      "epoch": 0.7290199652058653,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.046742730953258e-05,
+      "loss": 3.8366,
+      "step": 2200
+    },
+    {
+      "epoch": 0.7455886007787259,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.138755980861244e-05,
+      "loss": 3.8347,
+      "step": 2250
+    },
+    {
+      "epoch": 0.7621572363515865,
+      "grad_norm": 0.80078125,
+      "learning_rate": 4.230769230769231e-05,
+      "loss": 3.8299,
+      "step": 2300
+    },
+    {
+      "epoch": 0.778725871924447,
+      "grad_norm": 0.75390625,
+      "learning_rate": 4.3227824806772176e-05,
+      "loss": 3.8187,
+      "step": 2350
+    },
+    {
+      "epoch": 0.7952945074973076,
+      "grad_norm": 0.76953125,
+      "learning_rate": 4.4147957305852044e-05,
+      "loss": 3.8225,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8118631430701682,
+      "grad_norm": 0.81640625,
+      "learning_rate": 4.506808980493191e-05,
+      "loss": 3.8229,
+      "step": 2450
+    },
+    {
+      "epoch": 0.8284317786430287,
+      "grad_norm": 0.77734375,
+      "learning_rate": 4.598822230401178e-05,
+      "loss": 3.8245,
+      "step": 2500
+    },
+    {
+      "epoch": 0.8450004142158893,
+      "grad_norm": 0.78515625,
+      "learning_rate": 4.690835480309165e-05,
+      "loss": 3.8109,
+      "step": 2550
+    },
+    {
+      "epoch": 0.8615690497887499,
+      "grad_norm": 0.78125,
+      "learning_rate": 4.782848730217152e-05,
+      "loss": 3.8195,
+      "step": 2600
+    },
+    {
+      "epoch": 0.8781376853616105,
+      "grad_norm": 0.7890625,
+      "learning_rate": 4.874861980125138e-05,
+      "loss": 3.8125,
+      "step": 2650
+    },
+    {
+      "epoch": 0.894706320934471,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.966875230033125e-05,
+      "loss": 3.8131,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9112749565073316,
+      "grad_norm": 0.73046875,
+      "learning_rate": 4.999998362078322e-05,
+      "loss": 3.819,
+      "step": 2750
+    },
+    {
+      "epoch": 0.9278435920801922,
+      "grad_norm": 0.75,
+      "learning_rate": 4.999989244747393e-05,
+      "loss": 3.8082,
+      "step": 2800
+    },
+    {
+      "epoch": 0.9444122276530528,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.9999721297876855e-05,
+      "loss": 3.8146,
+      "step": 2850
+    },
+    {
+      "epoch": 0.9609808632259134,
+      "grad_norm": 0.765625,
+      "learning_rate": 4.999947017253951e-05,
+      "loss": 3.8042,
+      "step": 2900
+    },
+    {
+      "epoch": 0.977549498798774,
+      "grad_norm": 0.734375,
+      "learning_rate": 4.9999139072265274e-05,
+      "loss": 3.8072,
+      "step": 2950
+    },
+    {
+      "epoch": 0.9941181343716345,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.9998727998113335e-05,
+      "loss": 3.8008,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0106039267666307,
+      "grad_norm": 0.31640625,
+      "learning_rate": 4.999823695139877e-05,
+      "loss": 3.7924,
+      "step": 3050
+    },
+    {
+      "epoch": 1.0271725623394914,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.999766593369246e-05,
+      "loss": 3.7963,
+      "step": 3100
+    },
+    {
+      "epoch": 1.0437411979123519,
+      "grad_norm": 0.33203125,
+      "learning_rate": 4.999701494682112e-05,
+      "loss": 3.7837,
+      "step": 3150
+    },
+    {
+      "epoch": 1.0603098334852126,
+      "grad_norm": 0.298828125,
+      "learning_rate": 4.999628399286731e-05,
+      "loss": 3.7942,
+      "step": 3200
+    },
+    {
+      "epoch": 1.076878469058073,
+      "grad_norm": 0.298828125,
+      "learning_rate": 4.99954730741694e-05,
+      "loss": 3.7819,
+      "step": 3250
+    },
+    {
+      "epoch": 1.0934471046309335,
+      "grad_norm": 0.296875,
+      "learning_rate": 4.999458219332157e-05,
+      "loss": 3.7868,
+      "step": 3300
+    },
+    {
+      "epoch": 1.1100157402037942,
+      "grad_norm": 0.294921875,
+      "learning_rate": 4.9993611353173794e-05,
+      "loss": 3.7924,
+      "step": 3350
+    },
+    {
+      "epoch": 1.1265843757766547,
+      "grad_norm": 0.29296875,
+      "learning_rate": 4.999256055683187e-05,
+      "loss": 3.7884,
+      "step": 3400
+    },
+    {
+      "epoch": 1.1431530113495154,
+      "grad_norm": 0.287109375,
+      "learning_rate": 4.999142980765736e-05,
+      "loss": 3.7875,
+      "step": 3450
+    },
+    {
+      "epoch": 1.159721646922376,
+      "grad_norm": 0.3125,
+      "learning_rate": 4.9990219109267596e-05,
+      "loss": 3.7827,
+      "step": 3500
+    },
+    {
+      "epoch": 1.1762902824952366,
+      "grad_norm": 0.314453125,
+      "learning_rate": 4.9988928465535686e-05,
+      "loss": 3.7832,
+      "step": 3550
+    },
+    {
+      "epoch": 1.192858918068097,
+      "grad_norm": 0.28515625,
+      "learning_rate": 4.9987557880590486e-05,
+      "loss": 3.7854,
+      "step": 3600
+    },
+    {
+      "epoch": 1.2094275536409578,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.998610735881659e-05,
+      "loss": 3.7765,
+      "step": 3650
+    },
+    {
+      "epoch": 1.2259961892138183,
+      "grad_norm": 0.27734375,
+      "learning_rate": 4.99845769048543e-05,
+      "loss": 3.7835,
+      "step": 3700
+    },
+    {
+      "epoch": 1.2425648247866787,
+      "grad_norm": 0.298828125,
+      "learning_rate": 4.998296652359965e-05,
+      "loss": 3.7809,
+      "step": 3750
+    },
+    {
+      "epoch": 1.2591334603595394,
+      "grad_norm": 0.310546875,
+      "learning_rate": 4.9981276220204344e-05,
+      "loss": 3.7849,
+      "step": 3800
+    },
+    {
+      "epoch": 1.2757020959324,
+      "grad_norm": 0.3125,
+      "learning_rate": 4.997950600007578e-05,
+      "loss": 3.7815,
+      "step": 3850
+    },
+    {
+      "epoch": 1.2922707315052606,
+      "grad_norm": 0.302734375,
+      "learning_rate": 4.997765586887702e-05,
+      "loss": 3.7793,
+      "step": 3900
+    },
+    {
+      "epoch": 1.308839367078121,
+      "grad_norm": 0.30859375,
+      "learning_rate": 4.997572583252672e-05,
+      "loss": 3.7729,
+      "step": 3950
+    },
+    {
+      "epoch": 1.3254080026509816,
+      "grad_norm": 0.29296875,
+      "learning_rate": 4.9973715897199226e-05,
+      "loss": 3.7745,
+      "step": 4000
+    },
+    {
+      "epoch": 1.3419766382238423,
+      "grad_norm": 0.32421875,
+      "learning_rate": 4.9971626069324435e-05,
+      "loss": 3.7688,
+      "step": 4050
+    },
+    {
+      "epoch": 1.358545273796703,
+      "grad_norm": 0.275390625,
+      "learning_rate": 4.996945635558785e-05,
+      "loss": 3.7748,
+      "step": 4100
+    },
+    {
+      "epoch": 1.3751139093695635,
+      "grad_norm": 0.291015625,
+      "learning_rate": 4.996720676293052e-05,
+      "loss": 3.7686,
+      "step": 4150
+    },
+    {
+      "epoch": 1.391682544942424,
+      "grad_norm": 0.287109375,
+      "learning_rate": 4.9964877298549045e-05,
+      "loss": 3.7736,
+      "step": 4200
+    },
+    {
+      "epoch": 1.4082511805152846,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.9962467969895535e-05,
+      "loss": 3.7751,
+      "step": 4250
+    },
+    {
+      "epoch": 1.4248198160881451,
+      "grad_norm": 0.2890625,
+      "learning_rate": 4.995997878467758e-05,
+      "loss": 3.7673,
+      "step": 4300
+    },
+    {
+      "epoch": 1.4413884516610058,
+      "grad_norm": 0.34375,
+      "learning_rate": 4.995740975085825e-05,
+      "loss": 3.7742,
+      "step": 4350
+    },
+    {
+      "epoch": 1.4579570872338663,
+      "grad_norm": 0.294921875,
+      "learning_rate": 4.9954760876656056e-05,
+      "loss": 3.7737,
+      "step": 4400
+    },
+    {
+      "epoch": 1.4745257228067268,
+      "grad_norm": 0.283203125,
+      "learning_rate": 4.995203217054493e-05,
+      "loss": 3.7704,
+      "step": 4450
+    },
+    {
+      "epoch": 1.4910943583795875,
+      "grad_norm": 0.294921875,
+      "learning_rate": 4.9949223641254156e-05,
+      "loss": 3.7693,
+      "step": 4500
+    },
+    {
+      "epoch": 1.507662993952448,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.994633529776842e-05,
+      "loss": 3.76,
+      "step": 4550
+    },
+    {
+      "epoch": 1.5242316295253087,
+      "grad_norm": 0.27734375,
+      "learning_rate": 4.994336714932771e-05,
+      "loss": 3.7617,
+      "step": 4600
+    },
+    {
+      "epoch": 1.5408002650981691,
+      "grad_norm": 0.287109375,
+      "learning_rate": 4.9940319205427335e-05,
+      "loss": 3.7737,
+      "step": 4650
+    },
+    {
+      "epoch": 1.5573689006710296,
+      "grad_norm": 0.310546875,
+      "learning_rate": 4.993719147581787e-05,
+      "loss": 3.7699,
+      "step": 4700
+    },
+    {
+      "epoch": 1.5739375362438903,
+      "grad_norm": 0.306640625,
+      "learning_rate": 4.9933983970505116e-05,
+      "loss": 3.7665,
+      "step": 4750
+    },
+    {
+      "epoch": 1.590506171816751,
+      "grad_norm": 0.30859375,
+      "learning_rate": 4.9930696699750095e-05,
+      "loss": 3.7622,
+      "step": 4800
+    },
+    {
+      "epoch": 1.6070748073896115,
+      "grad_norm": 0.287109375,
+      "learning_rate": 4.992732967406901e-05,
+      "loss": 3.7572,
+      "step": 4850
+    },
+    {
+      "epoch": 1.623643442962472,
+      "grad_norm": 0.3125,
+      "learning_rate": 4.992388290423318e-05,
+      "loss": 3.7626,
+      "step": 4900
+    },
+    {
+      "epoch": 1.6402120785353325,
+      "grad_norm": 0.328125,
+      "learning_rate": 4.9920356401269055e-05,
+      "loss": 3.7626,
+      "step": 4950
+    },
+    {
+      "epoch": 1.6567807141081932,
+      "grad_norm": 0.31640625,
+      "learning_rate": 4.991675017645815e-05,
+      "loss": 3.7626,
+      "step": 5000
+    },
+    {
+      "epoch": 1.6567807141081932,
+      "eval_loss": 3.72454833984375,
+      "eval_runtime": 7.9243,
+      "eval_samples_per_second": 122.282,
+      "eval_steps_per_second": 2.019,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 90540,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 30,
+  "save_steps": 5000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.4761889824453427e+19,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-5000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28bc9829b8c8fa8bf181924e8f4aa71e88dec472de000157e8bb54e4440b7211
+size 5841