Upload 9 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

NanoHammerForCausalLM.py

@@ -0,0 +1,812 @@
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+from typing import Optional, Tuple, Union, List
+from dataclasses import dataclass
+
+from transformers import PretrainedConfig
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast, BaseModelOutputWithPast
+from transformers.models.llama.modeling_llama import (
+    LlamaRMSNorm,
+    LlamaMLP,
+    LlamaAttention,
+    LlamaRotaryEmbedding,
+    apply_rotary_pos_emb,
+)
+
+
+# ============================================================================
+# Configuration
+# ============================================================================
+
+class NanoHammerConfig(PretrainedConfig):
+    model_type = "nanohammer"
+
+    def __init__(
+        self,
+        vocab_size=128256,
+        hidden_size=2048,
+        intermediate_size=8192,
+        num_hidden_layers=24,
+        num_attention_heads=32,
+        num_key_value_heads=8,  # GQA
+        num_state_heads=32,
+        state_hidden_size=None,  # 默认为 hidden_size * 4
+        max_position_embeddings=131072,
+        rms_norm_eps=1e-5,
+        initializer_range=0.02,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        mlp_bias=False,
+        hidden_act="silu",
+        # NanoHammer 特有参数
+        bos_token_id=128000,
+        eos_token_id=128009,
+        pad_token_id=None,
+        **kwargs
+    ):
+        # 设置 auto_map 以支持 trust_remote_code
+        if "auto_map" not in kwargs:
+            kwargs["auto_map"] = {
+                "AutoConfig": "NanoHammerForCausalLM.NanoHammerConfig",
+                "AutoModelForCausalLM": "NanoHammerForCausalLM.NanoHammerForCausalLM",
+            }
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.num_state_heads = num_state_heads
+
+        # 积分状态维度：默认为 hidden_size / 4，提升状态表征能力
+        self.state_hidden_size = state_hidden_size if state_hidden_size is not None else hidden_size / 4
+
+        self.max_position_embeddings = max_position_embeddings
+        self.rms_norm_eps = rms_norm_eps
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.mlp_bias = mlp_bias
+        self.hidden_act = hidden_act
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            **kwargs
+        )
+
+
+# ============================================================================
+# 创新组件 1: 全息旋转位置编码
+# ============================================================================
+
+class HolographicRotaryEmbedding(nn.Module):
+    """
+    全息旋转位置编码 - 为积分状态注入时间特征
+
+    核心思想：
+    - 对每个位置 i，应用复数域旋转：x_i * e^(i*θ_k)
+    - 积分后：S_t = Σ(x_i * e^(i*θ_k))，状态成为"多项式系数容器"
+    - 通过逆旋转 R_{-t} 转换为相对坐标系，实现平移不变性
+
+    关键修正：使用绝对 position_ids 而非相对 seq_len
+    """
+    def __init__(self, dim, max_position_embeddings=131072, base=10000):
+        super().__init__()
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+
+        # 计算频率：θ_k = base^(-2k/d)
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, x, position_ids):
+        """
+        应用旋转位置编码（使用绝对位置）
+
+        Args:
+            x: (B, T, D) - 输入张量
+            position_ids: (B, T) - 绝对位置索引
+        Returns:
+            x_rotated: (B, T, D) - 应用旋转编码后的张量
+        """
+        # position_ids: (B, T) -> (B, T, 1)
+        # inv_freq: (D/2,) -> (1, 1, D/2)
+        # freqs: (B, T, D/2)
+        freqs = torch.einsum("bt,d->btd", position_ids.to(x.dtype), self.inv_freq.to(x.dtype))
+
+        # 计算 cos 和 sin：(B, T, D/2)
+        cos = freqs.cos()
+        sin = freqs.sin()
+
+        # 扩展到完整维度：(B, T, D)
+        cos = torch.cat([cos, cos], dim=-1)
+        sin = torch.cat([sin, sin], dim=-1)
+
+        # 应用旋转变换
+        x1 = x[..., 0::2]
+        x2 = x[..., 1::2]
+
+        x1_rotated = x1 * cos[..., 0::2] - x2 * sin[..., 0::2]
+        x2_rotated = x1 * sin[..., 1::2] + x2 * cos[..., 1::2]
+
+        x_rotated = torch.stack([x1_rotated, x2_rotated], dim=-1).flatten(-2)
+        return x_rotated
+
+    def apply_inverse_rotation(self, x, position_ids):
+        """
+        应用逆旋转，转换为相对坐标系（使用绝对位置）
+
+        核心：S_t' = S_t * e^(-t*θ)，将积分状态转换为相对视角
+
+        Args:
+            x: (B, T, D) - 积分状态张量
+            position_ids: (B, T) - 绝对位置索引
+        Returns:
+            x_relative: (B, T, D) - 相对坐标系下的状态
+        """
+        # 使用绝对位置计算逆旋转
+        freqs = torch.einsum("bt,d->btd", position_ids.to(x.dtype), self.inv_freq.to(x.dtype))
+
+        # 计算 cos(-θ) 和 sin(-θ)
+        cos = freqs.cos()
+        sin = -freqs.sin()  # 负号！逆旋转的关键
+
+        cos = torch.cat([cos, cos], dim=-1)
+        sin = torch.cat([sin, sin], dim=-1)
+
+        # 应用逆旋转
+        x1 = x[..., 0::2]
+        x2 = x[..., 1::2]
+
+        x1_relative = x1 * cos[..., 0::2] + x2 * sin[..., 0::2]
+        x2_relative = -x1 * sin[..., 1::2] + x2 * cos[..., 1::2]
+
+        x_relative = torch.stack([x1_relative, x2_relative], dim=-1).flatten(-2)
+        return x_relative
+
+
+# ============================================================================
+# 创新组件 2: 多头状态更新单元
+# ============================================================================
+
+class StateUpdateCell(nn.Module):
+    """
+    Multi-Head State Update Cell - 欧拉法固定点迭代
+
+    在全息积分状态上进行非线性演化：
+    - S_{t+1} = S_t + α·f(S_t)
+    - 每个头在独立子空间迭代
+    - 可学习步长 α
+    """
+    def __init__(self, config):
+        super().__init__()
+        self.hidden_size = config.state_hidden_size
+        self.num_heads = config.num_state_heads
+        self.head_dim = config.state_hidden_size // config.num_state_heads
+
+        assert config.state_hidden_size % config.num_state_heads == 0
+
+        # Pre-Norm
+        self.pre_norm = LlamaRMSNorm(config.state_hidden_size, eps=config.rms_norm_eps)
+
+        # MLP 更新函数
+        self.mlp = nn.Sequential(
+            nn.Linear(config.state_hidden_size, config.state_hidden_size * 4, bias=False),
+            nn.SiLU(),
+            nn.Linear(config.state_hidden_size * 4, config.state_hidden_size, bias=False)
+        )
+
+        # Post-Norm
+        self.post_norm = LlamaRMSNorm(config.state_hidden_size, eps=config.rms_norm_eps)
+
+        # 欧拉法步长（可学习，每个头独立）
+        self.step_size = nn.Parameter(torch.ones(self.num_heads) * 0.1)
+
+    def forward(self, state):
+        """
+        欧拉法更新：S_{t+1} = S_t + α * f(S_t)
+
+        Args:
+            state: (B, T, state_hidden_size)
+        Returns:
+            state: (B, T, state_hidden_size)
+        """
+        batch_size, seq_len, _ = state.shape
+
+        # Pre-Norm
+        state_normed = self.pre_norm(state)
+
+        # MLP 计算增量
+        delta = self.mlp(state_normed)  # (B, T, state_hidden_size)
+
+        # Reshape 为多头
+        state_heads = state.view(batch_size, seq_len, self.num_heads, self.head_dim)
+        delta_heads = delta.view(batch_size, seq_len, self.num_heads, self.head_dim)
+
+        # 每个头独立步长更新
+        step_size = self.step_size.view(1, 1, self.num_heads, 1)
+        state_heads = state_heads + step_size * delta_heads
+
+        # Merge heads
+        state = state_heads.view(batch_size, seq_len, self.hidden_size)
+
+        # Post-Norm
+        state = self.post_norm(state)
+        return state
+
+
+# ============================================================================
+# 创新组件 4: State Token 投影
+# ============================================================================
+
+class StateTokenProjection(nn.Module):
+    """
+    State -> Token 投影：将全息积分状态投影为 hidden_size 维度的 token
+
+    这个 state token 将被添加到序列开头，参与标准的 Llama attention
+    """
+    def __init__(self, config):
+        super().__init__()
+        self.state_to_hidden = nn.Linear(config.state_hidden_size, config.hidden_size, bias=False)
+        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(self, state: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            state: (B, T, state_hidden_size) - 全息积分状态
+        Returns:
+            state_token: (B, T, hidden_size) - 投影后的 state token
+        """
+        state_token = self.state_to_hidden(state)
+        state_token = self.norm(state_token)
+        return state_token
+
+
+# ============================================================================
+# 创新组件 5: 混合 Decoder Layer（State Token + Llama Attention）
+# ============================================================================
+
+class HybridNanoHammerDecoderLayer(nn.Module):
+    """
+    NanoHammer 解码器层：因果 State Tokens 前缀 + 标准 Llama Attention + MLP
+
+    流程：
+    1. State 更新：非线性演化全息积分状态
+    2. 因果 State Tokens 生成：
+       - 对于位置 i，生成 state_token_i（包含截止到 i-1 的累积状态）
+       - 所有 state tokens 作为前缀：[state_0, state_1, ..., state_{T-1}, hidden_0, ..., hidden_{T-1}]
+       - 序列长度变为 2T
+    3. 特殊 Attention Mask：
+       - hidden_i 可以 attend 到：state_j (j <= i) 和 hidden_j (j < i)
+       - 确保因果性：位置 i 只能看到它之前的历史状态
+    4. Self-Attention：标准 Llama attention（序列长度 = 2T）
+    5. MLP：前馈网络
+    6. 移除 state tokens：只返回 hidden states 部分
+
+    关键设计：
+    - State tokens 作为"全局前缀"，每个对应一个位置的历史
+    - 通过精心设计的 attention mask 确保因果性
+    - 优雅地保留了"state token 在前"的架构理念
+    """
+    def __init__(self, config, layer_idx: int, holographic_rope: HolographicRotaryEmbedding):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.layer_idx = layer_idx
+        self.holographic_rope = holographic_rope
+
+        # 0. State 更新单元
+        self.state_cell = StateUpdateCell(config)
+
+        # 1. State Token 投影
+        self.state_projection = StateTokenProjection(config)
+
+        # 2. Self-Attention（标准 Llama）
+        self.self_attn = LlamaAttention(config, layer_idx)
+        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        # Llama RoPE for position embeddings
+        self.rotary_emb = LlamaRotaryEmbedding(config=config)
+
+        # 3. MLP
+        self.mlp = LlamaMLP(config)
+        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        state: torch.Tensor,
+        position_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        **kwargs
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Args:
+            hidden_states: (B, T, hidden_size) - 输入token表示
+            state: (B, T_state, state_hidden_size) - 全息积分累积和
+            position_ids: (B, T_full) - 完整序列的位置索引
+            attention_mask: (B, 1, T, T) - 输入序列的因果mask
+        Returns:
+            hidden_states: (B, T, hidden_size) - 输出token表示
+            state: (B, T_state, state_hidden_size) - 更新后的状态
+        """
+        batch_size, seq_len, _ = hidden_states.shape
+
+        # 0. State 非线性更新
+        state = self.state_cell(state)
+
+        # 1. 为每个位置创建因果 state（在绝对坐标系）
+        # 对于位置 i，生成 state_token_i（使用截止到 i-1 的累积状态）
+        if position_ids is None:
+            raise ValueError("position_ids is required for inverse rotation")
+
+        state_shifted = torch.cat([
+            torch.zeros_like(state[:, :1, :]),  # 位置 0 没有历史状态
+            state[:, :-1, :]  # 位置 i 使用截止到 i-1 的累积状态（绝对坐标系）
+        ], dim=1)  # (B, T_state, state_hidden_size)
+
+        # 2. 对平移后的 state 应用逆旋转（转换为相对坐标系）
+        state_relative = self.holographic_rope.apply_inverse_rotation(state_shifted, position_ids)
+
+        # 3. 投影为 state tokens
+        state_tokens = self.state_projection(state_relative)  # (B, T_state, hidden_size)
+
+        # 4. 只使用与当前输入对应的 state tokens
+        if state_tokens.shape[1] > seq_len:
+            state_tokens = state_tokens[:, -seq_len:, :]  # (B, T, hidden_size)
+
+        # 5. 将 state tokens 作为前缀拼接到序列开头
+        # 序列结构：[state_0, state_1, ..., state_{T-1}, hidden_0, hidden_1, ..., hidden_{T-1}]
+        combined_input = torch.cat([state_tokens, hidden_states], dim=1)  # (B, 2T, hidden_size)
+
+        # 6. 准备扩展的 position_ids
+        # State tokens 使用与对应 hidden token 相同的位置编码
+        if position_ids.shape[1] > seq_len:
+            current_position_ids = position_ids[:, -seq_len:]  # (B, T)
+        else:
+            current_position_ids = position_ids  # (B, T)
+
+        # 扩展 position_ids：[pos_0, pos_1, ..., pos_{T-1}, pos_0, pos_1, ..., pos_{T-1}]
+        extended_position_ids = torch.cat([current_position_ids, current_position_ids], dim=1)  # (B, 2T)
+
+        # 7. 构建特殊的因果 attention mask
+        # 关键设计：hidden_i 可以 attend 到 state_j (j <= i) 和 hidden_j (j < i)
+        extended_mask = torch.full(
+            (batch_size, 1, 2 * seq_len, 2 * seq_len),
+            torch.finfo(combined_input.dtype).min,
+            dtype=combined_input.dtype,
+            device=combined_input.device
+        )
+
+        # 构建 mask 矩阵 (2T x 2T)：
+        # - 前 T 行（state tokens）：每�� state_i 可以看到所有之前的 state（但这部分实际不会被使用）
+        # - 后 T 行（hidden tokens）：
+        #   - hidden_i 可以看到 state_j (j <= i)：列 0 到 i
+        #   - hidden_i 可以看到 hidden_j (j < i)：列 T 到 T+i-1
+
+        # State tokens 部分的 mask（上半部分，实际不重要，因为我们最后会丢弃）
+        for i in range(seq_len):
+            # state_i 可以看到 state_j (j <= i)
+            extended_mask[:, :, i, :i+1] = 0
+
+        # Hidden tokens 部分的 mask（下半部分，关键部分）
+        for i in range(seq_len):
+            # hidden_i 可以看到 state_j (j <= i)
+            extended_mask[:, :, seq_len + i, :i+1] = 0
+            # hidden_i 可以看到 hidden_j (j < i)
+            if i > 0:
+                extended_mask[:, :, seq_len + i, seq_len:seq_len+i] = 0
+
+        # 8. Self-Attention（序列长度为 2T）
+        residual = combined_input
+        hidden_states_normed = self.input_layernorm(combined_input)
+
+        # 生成 position embeddings（RoPE）
+        position_embeddings = self.rotary_emb(hidden_states_normed, extended_position_ids)
+
+        attn_output = self.self_attn(
+            hidden_states_normed,
+            attention_mask=extended_mask,
+            position_ids=extended_position_ids,
+            position_embeddings=position_embeddings,
+        )[0]  # (B, 2T, hidden_size)
+
+        combined_output = residual + attn_output
+
+        # 9. MLP
+        residual = combined_output
+        combined_output = self.post_attention_layernorm(combined_output)
+        combined_output = self.mlp(combined_output)
+        combined_output = residual + combined_output  # (B, 2T, hidden_size)
+
+        # 10. 移除 state tokens，只返回 hidden states 部分
+        hidden_states = combined_output[:, seq_len:, :]  # (B, T, hidden_size)
+
+        return hidden_states, state
+
+
+# ============================================================================
+# 主模型
+# ============================================================================
+
+class NanoHammerPreTrainedModel(PreTrainedModel):
+    config_class = NanoHammerConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["HybridNanoHammerDecoderLayer"]
+    _skip_keys_device_placement = ["past_key_values"]
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+class NanoHammerModel(NanoHammerPreTrainedModel):
+    """
+    NanoHammer 主模型
+
+    核心架构：
+    1. Token Embedding + State 初始化（全息积分）
+    2. N × HybridDecoderLayer（State Update + Self-Attn + Cross-Attn + MLP）
+    3. Final Norm
+    """
+    def __init__(self, config: NanoHammerConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        # Token Embedding
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+
+        # Token -> State 投影
+        self.token_to_state = nn.Linear(config.hidden_size, config.state_hidden_size, bias=False)
+
+        # 全息旋转位置编码
+        self.holographic_rope = HolographicRotaryEmbedding(
+            config.state_hidden_size,
+            max_position_embeddings=config.max_position_embeddings
+        )
+
+        # Decoder Layers
+        self.layers = nn.ModuleList([
+            HybridNanoHammerDecoderLayer(config, layer_idx, self.holographic_rope)
+            for layer_idx in range(config.num_hidden_layers)
+        ])
+
+        # Final Norm
+        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,  # 未使用，兼容接口
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # 1. Token Embedding
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        batch_size, seq_len, _ = inputs_embeds.shape
+        device = inputs_embeds.device
+
+        # 2. 初始化 State
+        # 重要设计决策：
+        # - past_state_absolute 存储的是 **绝对坐标系** 下的累积和（方便继续累积）
+        past_state_absolute = kwargs.get("past_state_absolute", None)
+
+        # Token -> State 投影
+        state_input = self.token_to_state(inputs_embeds)
+
+        # 3. Position IDs（必须在旋转编码之前生成）
+        # 计算 past_length
+        past_length = 0
+        if past_state_absolute is not None:
+            past_length = past_state_absolute.shape[1]
+
+        if position_ids is None:
+            # 生成完整的 position_ids（从 0 到 past_length+seq_len-1）
+            full_position_ids = torch.arange(
+                0, past_length + seq_len, dtype=torch.long, device=device
+            )
+            full_position_ids = full_position_ids.unsqueeze(0).expand(batch_size, -1)  # (B, T_total)
+            # 旋转编码使用新输入对应的位置（从 past_length 开始）
+            position_ids = full_position_ids[:, past_length:]  # (B, T)
+        else:
+            # 用户提供了 position_ids，假设它对应新输入
+            # 构建完整的 position_ids：需要知道过去的位置，假设为连续
+            # 如果 past_state_absolute 存在，则 position_ids 应覆盖所有位置
+            # 这里简化：假设 position_ids 已经是完整的
+            full_position_ids = position_ids
+            # 检查长度是否匹配 past_length + seq_len
+            if full_position_ids.shape[1] != past_length + seq_len:
+                raise ValueError(
+                    f"position_ids length ({full_position_ids.shape[1]}) does not match "
+                    f"past_length+seq_len ({past_length}+{seq_len})"
+                )
+
+        # 应用全息旋转编码（使用绝对位置）
+        state_input_rotated = self.holographic_rope(state_input, position_ids)
+
+        # 全息积分：S_t = S_{t-1} + x_t * e^(t*θ)
+        if past_state_absolute is not None:
+            # 增量推理：从 past_state_absolute 继续累积
+            # past_state_absolute 是绝对坐标系下的累积和（已经过 StateUpdate）
+            last_cumsum_absolute = past_state_absolute[:, -1:, :]  # (B, 1, state_hidden_size)
+
+            # 从 last_cumsum_absolute 继续累积新输入
+            cumsum_offsets = torch.cumsum(state_input_rotated, dim=1)  # (B, T_new, state_hidden_size)
+            new_cumsum_absolute = last_cumsum_absolute + cumsum_offsets  # (B, T_new, state_hidden_size)
+
+            # 拼接完整的绝对坐标系 state
+            state = torch.cat([past_state_absolute, new_cumsum_absolute], dim=1)
+        else:
+            # 训练模式：从头开始累积
+            state = torch.cumsum(state_input_rotated, dim=1)  # (B, T, state_hidden_size)
+
+        # 4. 通过所有 Decoder Layers（直接使用绝对坐标系的 state）
+        hidden_states = inputs_embeds
+        all_hidden_states = () if output_hidden_states else None
+
+        # 4.5. 生成因果 attention mask（如果未提供）
+        if attention_mask is None:
+            # 创建标准因果 mask: (B, 1, seq_len, seq_len)
+            attention_mask = torch.zeros(
+                (batch_size, 1, seq_len, seq_len),
+                dtype=inputs_embeds.dtype,
+                device=device
+            )
+            # 上三角部分填充 -inf（不能看到未来）
+            causal_mask = torch.triu(
+                torch.ones((seq_len, seq_len), device=device),
+                diagonal=1
+            ).bool()
+            attention_mask[:, :, causal_mask] = torch.finfo(inputs_embeds.dtype).min
+        elif attention_mask.dim() == 2:
+            # 将 2D padding mask 转换为 4D causal mask
+            # attention_mask: (B, seq_len) -> (B, 1, seq_len, seq_len)
+            expanded_mask = torch.zeros(
+                (batch_size, 1, seq_len, seq_len),
+                dtype=inputs_embeds.dtype,
+                device=device
+            )
+            # 先应用因果 mask
+            causal_mask = torch.triu(
+                torch.ones((seq_len, seq_len), device=device),
+                diagonal=1
+            ).bool()
+            expanded_mask[:, :, causal_mask] = torch.finfo(inputs_embeds.dtype).min
+
+            # 再应用 padding mask
+            for b in range(batch_size):
+                for i in range(seq_len):
+                    if attention_mask[b, i] == 0:
+                        # 这个位置被 padding，所有其他位置都不能 attend 到它
+                        expanded_mask[b, 0, :, i] = torch.finfo(inputs_embeds.dtype).min
+
+            attention_mask = expanded_mask
+
+        for decoder_layer in self.layers:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                hidden_states, state = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    state,  # 绝对坐标系的 state
+                    full_position_ids,  # 传递完整的 position_ids 用于逆旋转
+                    attention_mask,  # 传递 attention_mask
+                )
+            else:
+                hidden_states, state = decoder_layer(
+                    hidden_states,
+                    state,  # 绝对坐标系的 state
+                    full_position_ids,  # 传递完整的 position_ids 用于逆旋转
+                    attention_mask,  # 传递 attention_mask
+                )
+
+        # 5. Final Norm
+        hidden_states = self.norm(hidden_states)
+
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        # 6. 返回
+        # 直接返回 state（绝对坐标系），无需转换
+        state_to_cache = state if use_cache else None
+
+        if not return_dict:
+            outputs = (hidden_states,)
+            if output_hidden_states:
+                outputs += (all_hidden_states,)
+            # 将 state 作为额外的输出
+            outputs += (state_to_cache,)
+            return outputs
+
+        # 使用 BaseModelOutputWithPast，将 state 通过 attentions 字段传递
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=None,  # 不使用 KV cache
+            hidden_states=all_hidden_states,
+            attentions=(state_to_cache,) if state_to_cache is not None else None,  # 存放 state
+        )
+
+
+class NanoHammerForCausalLM(NanoHammerPreTrainedModel):
+    """
+    NanoHammer 语言模型（带 LM Head）
+    """
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = NanoHammerModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.model.embed_tokens.weight
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_state_absolute: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs  # 接收额外参数（例如来自 trainer 的参数）
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # 前向传播
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+            past_state_absolute=past_state_absolute,  # 传递 past_state_absolute
+        )
+
+        hidden_states = outputs.last_hidden_state
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        # 提取 state（从 attentions 字段）
+        state = outputs.attentions[0] if outputs.attentions is not None else None
+
+        # 计算损失
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + (outputs.hidden_states, state)
+            return ((loss,) + output) if loss is not None else output
+
+        # 返回时，将 state 通过自定义字段传递
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,  # 不再使用 KV cache
+            hidden_states=outputs.hidden_states,
+            attentions=(state,) if state is not None else None,  # State 通过 attentions 传递
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_state_absolute=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        **kwargs
+    ):
+        """
+        准备生成所需的输入（支持高效增量推理）
+
+        增量推理优化：
+        - past_state_absolute: NanoHammer 的全局 State cache（绝对坐标系）
+        - 当有 past_state_absolute 时，只传入最后一个 token
+        """
+        # 从上一步的输出提取 past_state_absolute
+        if past_state_absolute is None:
+            past_state_absolute = kwargs.get("past_state_absolute", None)
+
+        # 如果有 State cache，只需要最后一个 token（增量推理）
+        if past_state_absolute is not None:
+            input_ids = input_ids[:, -1:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_state_absolute is not None:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+
+        # 如果传递了 inputs_embeds，只在第一代使用
+        if inputs_embeds is not None and past_state_absolute is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_state_absolute": past_state_absolute,  # State cache
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    def _reorder_cache(past_state_absolute, beam_idx):
+        if past_state_absolute is None:
+            return None
+        return past_state_absolute.index_select(0, beam_idx.to(past_state_absolute.device))

README.md

@@ -1,3 +1,517 @@
----
-license: apache-2.0
----
+---
+language:
+- en
+license: apache-2.0
+base_model: meta-llama/Llama-3.2-1B-Instruct
+tags:
+- text-generation
+- causal-lm
+- transformers
+- nanohammer
+- holographic-embeddings
+- state-space
+- efficient-attention
+- long-context
+pipeline_tag: text-generation
+model-index:
+- name: NanoHammer-1.5B-Instruct
+  results:
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: AI2 Reasoning Challenge (ARC-Challenge)
+      type: arc_challenge
+    metrics:
+    - type: acc_norm
+      value: 33.28
+      name: normalized accuracy
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: AI2 Reasoning Challenge (ARC-Easy)
+      type: arc_easy
+    metrics:
+    - type: acc
+      value: 59.81
+      name: accuracy
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: HellaSwag
+      type: hellaswag
+    metrics:
+    - type: acc_norm
+      value: 56.33
+      name: normalized accuracy
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: PIQA
+      type: piqa
+    metrics:
+    - type: acc
+      value: 69.86
+      name: accuracy
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: WinoGrande
+      type: winogrande
+    metrics:
+    - type: acc
+      value: 57.14
+      name: accuracy
+---
+
+<div align="center">
+
+# 🔨 NanoHammer-1.5B-Instruct
+
+**Explicit Causal Modeling with Holographic Integral State Compression**
+
+*A novel hybrid architecture combining Transformer attention with O(1) global causal state*
+
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Model Size](https://img.shields.io/badge/Parameters-1.5B-green.svg)]()
+[![Context Length](https://img.shields.io/badge/Context-131K-orange.svg)]()
+
+</div>
+
+---
+
+## 🌟 Key Innovation: Explicit Causal Modeling
+
+NanoHammer introduces a **groundbreaking hybrid architecture** that augments standard Transformer layers with an **explicit causal state mechanism**. Unlike traditional attention that implicitly learns causal dependencies across O(n²) token pairs, NanoHammer maintains a **single global state token** that explicitly captures and propagates causal information through the sequence.
+
+### 🎯 Core Advantages
+
+| Feature | Traditional Attention | NanoHammer |
+|---------|---------------------|------------|
+| **Causal Modeling** | Implicit (learned) | **Explicit (structured)** |
+| **Global State Complexity** | O(n²) pairwise | **O(1) constant** |
+| **Extrapolation Cost** | Grows with sequence | **Constant O(1)** |
+| **Long Context Efficiency** | Quadratic scaling | **Linear scaling** |
+| **State Compression** | Distributed across KV cache | **Single token compression** |
+
+### 🔬 Technical Breakthrough
+
+```
+Traditional Transformer:     NanoHammer Architecture:
+Token₁ → Attention → Token₁' Token₁ ──→ State Update → S(t)
+Token₂ → Attention → Token₂'            ↓
+Token₃ → Attention → Token₃' [S(t)] + [Token₁...Tokenₙ] → Attention → Output
+  ...        O(n²)                    O(1)  +  O(n²)  =  O(n²)
+Tokenₙ → Attention → Tokenₙ'        But with global causal context!
+```
+
+The state token **S(t)** acts as a **causal information accumulator**, providing:
+- **Holographic encoding**: Position-aware via complex-domain rotations (e^(iθ))
+- **Fixed-point iteration**: Multi-head Euler method for stable state evolution
+- **Constant extrapolation**: New tokens always interact with O(1) state, not O(n) history
+
+---
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install transformers torch
+```
+
+### Basic Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+# Load model
+model_path = "NoesisLab/NanoHammer-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    trust_remote_code=True,
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+# Generate response
+prompt = "Explain the concept of causality in physics."
+messages = [{"role": "user", "content": prompt}]
+
+input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
+
+outputs = model.generate(
+    **inputs,
+    max_new_tokens=256,
+    temperature=0.7,
+    do_sample=True,
+    top_p=0.9,
+)
+
+response = tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
+print(response)
+```
+
+### Multi-turn Conversation
+
+```python
+messages = [
+    {"role": "user", "content": "What is a holographic state?"},
+    {"role": "assistant", "content": "A holographic state is a compressed representation that encodes global information..."},
+    {"role": "user", "content": "How does it differ from traditional attention?"}
+]
+
+input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+# ... generate as above
+```
+
+---
+
+## 🏗️ Architecture Details
+
+### Hybrid Decoder Layer Flow
+
+Each NanoHammer decoder layer executes the following pipeline:
+
+```
+Input Tokens (T tokens)
+    ↓
+[1] State Update Cell
+    • Multi-head fixed-point iteration: S_{t+1} = S_t + α·f(S_t)
+    • Learnable per-head step sizes
+    • Pre-norm → MLP → Post-norm
+    ↓
+[2] State Token Projection
+    • Project state_hidden_size (512) → hidden_size (2048)
+    • Create global "state token" encoding causal history
+    ↓
+[3] State Token Injection
+    • Prepend state token: [S(t)] + [Token₁, ..., Tokenₜ]
+    • Sequence length: T → T+1
+    ↓
+[4] Llama Self-Attention
+    • Standard Llama attention over T+1 tokens
+    • GQA: 32 query heads, 8 KV heads
+    • RoPE position encoding
+    ↓
+[5] Llama MLP
+    • SwiGLU activation
+    • 2048 → 8192 → 2048
+    ↓
+[6] State Token Removal
+    • Extract and remove state token
+    • Return T tokens
+    ↓
+Output Tokens (T tokens)
+```
+
+### Core Components
+
+#### 1️⃣ **HolographicRotaryEmbedding**
+```python
+# Complex-domain rotational encoding
+x_i * e^(i*θ_k)  where θ_k = position_id / (10000^(2k/d))
+```
+- Encodes **absolute positions** in complex space
+- Enables **inverse rotation** for relative coordinate transformations
+- Maintains **temporal coherence** across state updates
+
+#### 2️⃣ **StateUpdateCell**
+```python
+# Multi-head Euler iteration
+for head in range(num_state_heads):
+    S_new[head] = S[head] + step_size[head] * MLP(LayerNorm(S[head]))
+```
+- **16 independent state heads** (512-dim total)
+- **Learnable step sizes** per head for adaptive evolution
+- **Pre-norm + MLP + Post-norm** architecture for stability
+
+#### 3️⃣ **StateTokenProjection**
+```python
+# Compress global state into single token
+state_token = Linear(state_hidden_size=512 → hidden_size=2048)
+```
+- **Dimensional expansion**: 512 → 2048
+- **Single token** represents entire causal history
+- **O(1) memory footprint** regardless of sequence length
+
+### Model Specifications
+
+| Parameter | Value |
+|-----------|-------|
+| **Total Parameters** | ~1.5B |
+| **Hidden Size** | 2048 |
+| **Intermediate Size** | 8192 |
+| **Num Layers** | 16 |
+| **Attention Heads** | 32 (query) / 8 (KV, GQA) |
+| **State Heads** | 16 |
+| **State Hidden Size** | 512 |
+| **Vocab Size** | 128,256 |
+| **Max Position Embeddings** | 131,072 |
+| **RoPE Theta** | 500,000 |
+
+---
+
+## ⚡ Performance Characteristics
+
+### Computational Complexity
+
+| Operation | Complexity | Description |
+|-----------|-----------|-------------|
+| **State Update** | O(1) | Fixed-size state iteration |
+| **State Projection** | O(1) | Single token transformation |
+| **Self-Attention** | O(n²) | Standard Transformer attention |
+| **Total per Layer** | **O(n²)** | Dominated by attention (as expected) |
+
+**Key Insight**: While overall complexity remains O(n²) due to attention, the **state mechanism adds negligible overhead** while providing **explicit causal modeling** that is:
+- **Free during inference**: State update cost is independent of context length
+- **Efficient for extrapolation**: New tokens interact with O(1) state, not O(n) history
+- **Globally coherent**: Single state token ensures causal consistency
+
+### Memory Efficiency
+
+```
+Traditional KV Cache: O(n * d * L)  [n tokens × d dims × L layers]
+NanoHammer State:     O(d_s * L)    [512 dims × 16 layers = 8KB constant!]
+```
+
+The holographic state acts as a **learned compression** of causal history:
+- **Constant size** regardless of sequence length
+- **Accumulated knowledge** from all previous tokens
+- **Efficient transfer** across generation steps
+
+---
+
+## 📊 Benchmark Results
+
+NanoHammer has been evaluated on standard language understanding benchmarks using the [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) framework (0-shot evaluation).
+
+### Common Sense Reasoning & Knowledge
+
+| Task | Version | Metric | Value | Stderr |
+|------|---------|--------|-------|--------|
+| **ARC-Challenge** | 1 | acc | 29.61% | ±1.33% |
+| | | acc_norm | **33.28%** | ±1.38% |
+| **ARC-Easy** | 1 | acc | **59.81%** | ±1.01% |
+| | | acc_norm | 55.68% | ±1.02% |
+| **HellaSwag** | 1 | acc | 42.65% | ±0.49% |
+| | | acc_norm | **56.33%** | ±0.49% |
+| **PIQA** | 1 | acc | **69.86%** | ±1.07% |
+| | | acc_norm | **69.86%** | ±1.07% |
+| **WinoGrande** | 1 | acc | **57.14%** | ±1.39% |
+
+### Performance Summary
+
+```
+Average Accuracy (normalized): 54.86%
+- Strong performance on physical reasoning (PIQA: 69.86%)
+- Competitive commonsense reasoning (HellaSwag: 56.33%, WinoGrande: 57.14%)
+- Moderate performance on knowledge-intensive tasks (ARC: 33-60%)
+```
+
+**Key Observations:**
+- The model demonstrates **strong physical and commonsense reasoning** capabilities despite the novel architecture
+- Performance is competitive with other 1-2B parameter models in the same class
+- The explicit causal state mechanism does not compromise standard language understanding benchmarks
+- Results suggest the holographic state successfully captures relevant semantic information
+
+### Evaluation Details
+
+**Setup:**
+- Evaluation framework: `lm-evaluation-harness`
+- Shot configuration: 0-shot (no few-shot examples)
+- Temperature: Greedy decoding
+- Batch size: Auto
+
+**Reproducing Results:**
+```bash
+# Install lm-eval
+pip install lm-eval
+
+# Run evaluation
+lm_eval --model hf \
+    --model_args pretrained=NoesisLab/NanoHammer-1.5B-Instruct,trust_remote_code=True \
+    --tasks arc_challenge,arc_easy,hellaswag,piqa,winogrande \
+    --batch_size auto \
+    --output_path results/
+```
+
+---
+
+## 🎓 Training
+
+### Base Model & Weight Transfer
+
+NanoHammer initializes from **Llama-3.2-1B-Instruct** via selective weight transfer:
+
+**Frozen Components** (from Llama):
+- Token embeddings (`embed_tokens`)
+- Language modeling head (`lm_head`)
+- Self-attention layers (`self_attn`)
+- MLP layers (`mlp`)
+- All RMS layer norms
+
+**Trainable Components** (NanoHammer-specific):
+- `token_to_state`: Projects input tokens → state space
+- `holographic_rope`: Position encoding for state
+- `state_cell`: State update mechanism (per layer)
+- `state_projection`: State → hidden projection (per layer)
+
+### Training Configuration
+
+- **Dataset**: High-quality instruction-following data
+- **Precision**: BF16 mixed precision
+- **Optimization**: AdamW with cosine LR schedule
+- **Gradient Checkpointing**: Enabled for memory efficiency
+- **Batch Size**: Scaled with gradient accumulation
+- **Max Sequence Length**: 2048 tokens (extendable to 131K via RoPE)
+
+---
+
+## 🔍 Why NanoHammer?
+
+### Problem: Implicit vs Explicit Causal Modeling
+
+Traditional Transformers learn causal dependencies **implicitly** through attention weights:
+```
+Q @ K^T → Attention weights → Implicitly capture "what depends on what"
+```
+
+**Limitations**:
+- Causality is **distributed** across n² attention scores
+- **No explicit structure** for causal information flow
+- **Quadratic cost** to maintain global context
+- **Poor extrapolation** to longer sequences
+
+### Solution: Holographic Integral State
+
+NanoHammer introduces an **explicit causal state token**:
+```
+S(t) ← Accumulated causal information from all previous tokens
+     ← Updated via fixed-point iteration with temporal encoding
+     ← Participates in attention as a "global context token"
+```
+
+**Benefits**:
+- Causality is **explicit** in a structured state representation
+- **O(1) state size** provides constant-cost global context
+- **Natural extrapolation** to unseen sequence lengths
+- **Interpretable**: State token can be analyzed/visualized
+
+---
+
+## 📊 Model Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────┐
+│  Input: "What is the capital of France?"                │
+│  Tokens: [What, is, the, capital, of, France, ?]       │
+└────────────────┬────────────────────────────────────────┘
+                 │
+                 ▼
+         Token Embeddings
+                 │
+                 ▼
+    ┌────────────────────────┐
+    │  Token-to-State Proj   │  Project to state space
+    └────────────┬───────────┘
+                 │
+    ┌────────────▼───────────┐
+    │   Holographic RoPE     │  Apply position encoding
+    │   (Complex rotation)    │
+    └────────────┬───────────┘
+                 │
+         ╔═══════▼════════╗
+         ║   Layer 1-16   ║  (Repeated 16 times)
+         ╠════════════════╣
+         ║ ┌────────────┐ ║
+         ║ │State Update│ ║  S(t+1) = S(t) + α·f(S(t))
+         ║ │   Cell     │ ║  [Fixed-point iteration]
+         ║ └─────┬──────┘ ║
+         ║       │        ║
+         ║ ┌─────▼──────┐ ║
+         ║ │   State    │ ║  Project 512 → 2048
+         ║ │ Projection │ ║
+         ║ └─────┬──────┘ ║
+         ║       │        ║
+         ║   [S] + [T₁, T₂, ..., Tₙ]  ← Prepend state token
+         ║       │        ║
+         ║ ┌─────▼──────┐ ║
+         ║ │   Llama    │ ║  Standard attention
+         ║ │ Attention  │ ║  over T+1 tokens
+         ║ └─────┬──────┘ ║
+         ║       │        ║
+         ║ ┌─────▼──────┐ ║
+         ║ │   Llama    │ ║  SwiGLU MLP
+         ║ │    MLP     │ ║
+         ║ └─────┬──────┘ ║
+         ║       │        ║
+         ║   Remove [S] from output
+         ║       │        ║
+         ╚═══════▼════════╝
+                 │
+         ┌───────▼────────┐
+         │   Final Norm   │
+         └───────┬────────┘
+                 │
+         ┌───────▼────────┐
+         │     LM Head    │  Project to vocab
+         └───────┬────────┘
+                 │
+                 ▼
+    Output: "Paris" (logits over 128K vocab)
+```
+
+---
+
+## 📚 Citation
+
+If you use NanoHammer in your research, please cite:
+
+```bibtex
+@misc{nanohammer2025,
+  title={NanoHammer: Explicit Causal Modeling with Holographic Integral State Compression},
+  author={NoesisLab},
+  year={2025},
+  howpublished={\url{https://huggingface.co/NoesisLab/NanoHammer-1.5B-Instruct}},
+}
+```
+
+---
+
+## 📝 License
+
+This model is released under the **Apache 2.0** license, inheriting from the base Llama-3.2-1B-Instruct model.
+
+---
+
+## 🙏 Acknowledgments
+
+- **Base Model**: Meta's Llama-3.2-1B-Instruct
+- **Inspiration**: State-space models, holographic memory, and causal inference theory
+- **Framework**: HuggingFace Transformers
+
+---
+
+## 🔗 Links
+
+- **Model Card**: [NoesisLab/NanoHammer-1.5B-Instruct](https://huggingface.co/NoesisLab/NanoHammer-1.5B-Instruct)
+- **GitHub**: [NanoHammer Repository](https://github.com/NoesisLab/NanoHammer) *(if available)*
+- **Paper**: Coming soon
+
+---
+
+<div align="center">
+
+**Built with ❤️ by NoesisLab**
+
+*Advancing causal modeling in large language models*
+
+</div>

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "NanoHammerForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "NanoHammerForCausalLM.NanoHammerConfig",
+    "AutoModelForCausalLM": "NanoHammerForCausalLM.NanoHammerForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "nanohammer",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "num_state_heads": 16,
+  "pad_token_id": 128009,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "state_hidden_size": 512,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "vocab_size": 128256
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d0f17d08fcd74eb7b77937646e6e0202d2141de03f6dbedb27227854ae8aec3
+size 3099854832

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d44102a13ed5f1f80d7e5eb19eae6eee1c6ca395460da3e29c7c7fe5999a494
+size 6289