diff --git "a/reduced_ppl_average copy.json" "b/reduced_ppl_average copy.json" deleted file mode 100644--- "a/reduced_ppl_average copy.json" +++ /dev/null @@ -1,17965 +0,0 @@ -{ - "original_ppl": 3.697141408920288, - "reduced_ppl": [ - { - "bit": 1, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3464.481201171875 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3677.32177734375 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3220.70751953125 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 2131.33447265625 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3050.191650390625 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3169.390380859375 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 2524.9228515625 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3505.794189453125 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3512.329833984375 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3000.78466796875 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3446.185791015625 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3538.577880859375 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3145.047119140625 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3348.456787109375 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3593.623779296875 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3501.1298828125 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3036.5185546875 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3866.78857421875 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3348.3193359375 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3167.444580078125 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3323.9296875 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3516.482421875 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3708.910400390625 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 4347.1240234375 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3240.3759765625 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3352.1630859375 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3381.581298828125 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3230.160400390625 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3474.3310546875 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3624.54541015625 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3904.429443359375 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3931.96875 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3287.423583984375 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3556.094970703125 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3329.392578125 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3602.624267578125 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3520.4921875 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3685.15771484375 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3495.671630859375 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3414.18310546875 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3327.97998046875 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3443.551025390625 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3444.01416015625 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3339.546142578125 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 2957.610595703125 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3380.78125 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3418.731689453125 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3345.331787109375 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3653.060302734375 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3615.57275390625 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3484.28857421875 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3334.352783203125 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3082.19921875 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3499.62109375 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3387.6494140625 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3406.943115234375 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3531.309814453125 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3506.603515625 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3868.710205078125 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3764.70654296875 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3482.587890625 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3693.3271484375 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3579.640869140625 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3575.9287109375 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3613.32177734375 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3786.64208984375 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3601.834228515625 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3556.9462890625 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3534.129638671875 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3533.364501953125 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3586.906005859375 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3581.355224609375 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3393.70947265625 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3100.755615234375 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3498.159423828125 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3489.486328125 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3524.822509765625 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3541.474609375 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3562.73486328125 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3857.061279296875 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3233.695556640625 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3500.315185546875 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3616.383056640625 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3580.91796875 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3646.1826171875 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3648.7255859375 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3388.560546875 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3332.454833984375 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3620.230712890625 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3646.46435546875 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3618.018310546875 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3555.70849609375 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3578.958251953125 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3403.302734375 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3548.899169921875 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3505.10205078125 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3525.427734375 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3479.354736328125 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3556.078125 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3519.56591796875 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3695.744140625 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3425.1474609375 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3614.25927734375 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3449.5693359375 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3472.6650390625 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3519.465087890625 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3514.176025390625 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3397.32958984375 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3332.216796875 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3525.00732421875 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3529.767578125 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3414.82470703125 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3572.935546875 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3578.476806640625 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3620.157958984375 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3347.22412109375 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3481.14013671875 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3397.452880859375 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3501.714111328125 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3550.26318359375 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3554.250732421875 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3657.511962890625 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3503.755126953125 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3498.2060546875 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3591.5576171875 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3502.08154296875 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3557.892822265625 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3562.761962890625 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3387.365234375 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3597.06640625 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3500.82275390625 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3503.26416015625 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3435.848876953125 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3592.839111328125 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3547.57958984375 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3612.81884765625 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3530.255615234375 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3581.252685546875 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3535.090087890625 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3484.102783203125 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3616.645263671875 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3582.776123046875 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3615.041748046875 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3424.03369140625 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3534.27783203125 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3601.5146484375 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3554.345703125 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3570.53076171875 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3552.1259765625 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3440.91162109375 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3528.848876953125 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3523.068359375 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3497.592041015625 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3476.7373046875 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3579.920654296875 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3554.05078125 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3449.1318359375 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3632.650146484375 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3574.29541015625 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3606.746337890625 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3529.7841796875 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3531.518310546875 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3541.02197265625 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3489.4365234375 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3556.959716796875 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3601.68994140625 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3603.177734375 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3553.97265625 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3526.6650390625 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3569.91455078125 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3655.628662109375 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3565.99462890625 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3552.451171875 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3576.16064453125 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3590.4892578125 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3606.357666015625 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3585.305419921875 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3383.326171875 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3453.83544921875 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3570.1494140625 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3518.126220703125 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3598.716796875 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3585.339599609375 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3540.32666015625 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3647.01416015625 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3611.468505859375 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3496.431640625 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3465.0263671875 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3481.375732421875 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3568.396240234375 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3558.585205078125 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3567.113525390625 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3462.037109375 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3537.369873046875 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3454.96240234375 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3482.454833984375 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3580.498046875 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3562.57177734375 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3555.57958984375 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3659.325927734375 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3480.834716796875 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3484.60107421875 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3482.33544921875 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3526.134033203125 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3547.802978515625 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3615.197021484375 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3383.19384765625 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3525.115234375 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3534.8408203125 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3586.694091796875 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3581.105712890625 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3566.310791015625 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3430.217529296875 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3412.60107421875 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3544.95166015625 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3576.293701171875 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3588.49365234375 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3560.6220703125 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3558.55126953125 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3364.076416015625 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3563.0439453125 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3550.591796875 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3594.59716796875 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3594.569580078125 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3594.50146484375 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3576.072021484375 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3421.145263671875 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3577.722900390625 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3514.330322265625 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3542.4677734375 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3514.33349609375 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3542.069091796875 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3515.218505859375 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3684.802490234375 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3762.018310546875 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3573.9306640625 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3572.294921875 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3504.58056640625 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3569.55712890625 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3580.8291015625 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3705.78515625 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3521.28466796875 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3599.454833984375 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3466.814453125 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3495.891357421875 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3594.898681640625 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3569.083984375 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3679.317626953125 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3586.215087890625 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3479.630126953125 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3414.84423828125 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3582.878662109375 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3625.776123046875 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3583.28857421875 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3446.153076171875 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3570.9599609375 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3478.093994140625 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3520.408447265625 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3611.344482421875 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3527.7685546875 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3514.8330078125 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3553.281494140625 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3497.55224609375 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3538.7939453125 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3457.945556640625 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3492.0498046875 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3582.2705078125 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3596.208740234375 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3400.165771484375 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3518.549072265625 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3572.260986328125 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3459.627685546875 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3417.1669921875 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3565.590087890625 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3547.9111328125 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3573.610595703125 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3516.264404296875 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3507.37255859375 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3456.105712890625 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3545.1376953125 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3559.121337890625 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3523.32373046875 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3329.42138671875 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3736.813232421875 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3528.31689453125 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3544.701416015625 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3549.63037109375 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3636.865234375 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3560.995361328125 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3538.18994140625 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3502.953369140625 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3496.99853515625 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3557.852294921875 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3510.022705078125 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3564.0771484375 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3545.306640625 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3403.52978515625 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3596.146728515625 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3488.4248046875 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3589.684814453125 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3531.50146484375 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3578.11181640625 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3580.8837890625 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3546.92333984375 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3489.386474609375 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3459.037109375 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3493.471923828125 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3558.7548828125 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3516.304931640625 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3520.086181640625 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3563.96826171875 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3482.913330078125 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3543.123046875 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3548.310302734375 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3490.035400390625 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3539.091064453125 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3532.9736328125 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3556.837890625 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3466.3583984375 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3571.3857421875 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3528.11181640625 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3523.404296875 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3550.706787109375 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3552.03125 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3618.64306640625 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3700.209228515625 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3508.275634765625 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3543.61328125 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3558.97900390625 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3557.580810546875 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3568.529052734375 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3550.137939453125 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3618.56689453125 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3565.09375 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3551.69580078125 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3554.857421875 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3553.267822265625 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3556.234130859375 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3574.789794921875 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3567.1953125 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3521.724609375 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3572.42822265625 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3505.3828125 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3557.526611328125 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3562.70751953125 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3523.824462890625 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3506.0615234375 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3544.02880859375 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3531.734130859375 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3566.7734375 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3565.3994140625 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3555.77294921875 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3636.140380859375 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3626.523193359375 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3547.261474609375 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3565.2294921875 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3543.94775390625 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3546.78466796875 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3545.637939453125 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3511.783935546875 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3548.662109375 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3539.570068359375 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3569.689697265625 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3555.94921875 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3569.839599609375 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3568.117431640625 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3489.090087890625 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3524.620849609375 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3515.684326171875 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3541.211181640625 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3559.54248046875 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3559.8037109375 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3552.529296875 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3529.31298828125 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3501.303466796875 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3544.8671875 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3548.6689453125 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3551.418212890625 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3552.16015625 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3551.309814453125 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3567.399658203125 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3499.06689453125 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3537.140380859375 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3532.791748046875 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3530.0234375 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3560.214599609375 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3543.17724609375 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3515.684326171875 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3533.792724609375 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3533.68798828125 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3562.948974609375 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3537.25537109375 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3569.530029296875 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3553.112060546875 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3495.90478515625 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3468.282958984375 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3518.354248046875 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3540.329833984375 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3532.724365234375 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3551.790771484375 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3559.3115234375 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3564.862548828125 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3495.2314453125 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3538.206787109375 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3534.928466796875 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3577.497802734375 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3562.66015625 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3554.98291015625 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3525.93212890625 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3551.787353515625 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3512.072021484375 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3554.606689453125 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3531.20849609375 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3566.161376953125 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3554.548828125 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3594.090087890625 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3638.391357421875 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3517.961669921875 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3524.392333984375 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3545.21875 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3570.564697265625 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3556.678466796875 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3582.2021484375 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3548.87890625 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3518.823974609375 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3520.982421875 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3587.408935546875 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3551.90576171875 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3552.94580078125 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3533.260009765625 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3465.793212890625 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3572.6630859375 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3570.96337890625 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3549.400146484375 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3550.20263671875 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3552.08544921875 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3561.250244140625 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3524.570556640625 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3564.461181640625 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3547.227783203125 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3566.998046875 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3563.434814453125 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3560.856201171875 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3517.49560546875 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3594.71728515625 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3570.108642578125 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3522.974365234375 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3528.97314453125 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3553.93212890625 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3550.4091796875 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3491.064208984375 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3587.966552734375 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3556.359619140625 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3566.749755859375 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3528.774658203125 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3555.28466796875 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3557.7333984375 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3516.355224609375 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3582.998291015625 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3549.5625 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3512.741943359375 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3550.351318359375 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3539.79638671875 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3551.49951171875 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3550.679931640625 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3606.00341796875 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3535.595947265625 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3560.23486328125 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3591.150390625 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3569.308837890625 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3568.093505859375 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3533.839599609375 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3567.950439453125 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3572.162109375 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3518.13623046875 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3523.737060546875 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3541.10302734375 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3543.457763671875 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3527.93017578125 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3612.70166015625 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3546.033447265625 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3601.41845703125 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3519.5390625 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3547.81640625 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3554.748779296875 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3654.381103515625 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3527.40185546875 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3526.718994140625 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3566.67822265625 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3479.480712890625 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3577.4873046875 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3581.877685546875 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3606.828857421875 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3558.83642578125 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3533.543212890625 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3586.919921875 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3466.576416015625 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3552.190673828125 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3544.238525390625 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3628.2978515625 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3480.3369140625 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3502.919921875 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3559.270751953125 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3519.834228515625 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3542.278564453125 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3545.928955078125 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3588.63037109375 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3549.474609375 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3498.77001953125 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3538.935791015625 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3528.431396484375 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3553.376220703125 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3546.984130859375 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3502.365478515625 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3529.730712890625 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3485.910888671875 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3542.099365234375 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3547.71484375 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3530.568603515625 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3592.58203125 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3440.7705078125 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3562.411865234375 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3584.35498046875 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3511.000244140625 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3602.765380859375 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3510.27392578125 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3519.12255859375 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3594.0078125 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3542.25146484375 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3545.79345703125 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3633.08642578125 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3593.10302734375 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3584.33447265625 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3594.144775390625 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3612.37109375 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3576.371826171875 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3455.83544921875 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3628.678466796875 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3574.20361328125 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3504.771240234375 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3512.302978515625 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3581.53955078125 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3587.593505859375 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3645.18505859375 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3749.11376953125 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3590.612548828125 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3561.281005859375 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3498.61962890625 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3530.710205078125 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3567.9404296875 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3521.49609375 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3712.47412109375 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3666.239013671875 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3564.026123046875 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3569.373291015625 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3541.224609375 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3621.1181640625 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3426.92822265625 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3298.002685546875 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3188.660400390625 - } - ] - }, - { - "bit": 2, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 6.587299823760986 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 6.621645927429199 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 6.606184482574463 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 6.591529846191406 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 6.595300197601318 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 6.606298923492432 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 6.578024864196777 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 6.558347225189209 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 6.565422534942627 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 6.539042949676514 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 6.525199890136719 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 6.55272912979126 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 6.567893028259277 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 6.547338962554932 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 6.578011989593506 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 6.579082012176514 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 6.542652130126953 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 6.5580949783325195 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 6.531676769256592 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 6.560765743255615 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 6.541767120361328 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 6.579802989959717 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 6.601284503936768 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 6.570128917694092 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 6.569732189178467 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 6.539124488830566 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 6.561048984527588 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 6.572926998138428 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 6.5919389724731445 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 6.621241569519043 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 6.600897789001465 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 6.587358474731445 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 6.54921817779541 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 6.582107067108154 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 6.573081970214844 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 6.612077713012695 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 6.620278835296631 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 6.593531608581543 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 6.605550289154053 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 6.571804523468018 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 6.580696105957031 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 6.580711841583252 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 6.605426788330078 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 6.603277206420898 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 6.587345123291016 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 6.597412109375 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 6.5576558113098145 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 6.583559989929199 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 6.582685947418213 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 6.619879245758057 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 6.603657245635986 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 6.585909843444824 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 6.600263595581055 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 6.580042362213135 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 6.583291530609131 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 6.5865478515625 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 6.607184886932373 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 6.629411220550537 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 6.588752746582031 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 6.598271369934082 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 6.586933612823486 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 6.582657814025879 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 6.591380596160889 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 6.608002185821533 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 6.605777740478516 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 6.598511695861816 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 6.586959362030029 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 6.582998275756836 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 6.595479488372803 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 6.591337203979492 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 6.6023688316345215 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 6.600842475891113 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 6.5933027267456055 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 6.587803840637207 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 6.579985618591309 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 6.587390422821045 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 6.576834201812744 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 6.60318660736084 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 6.603156089782715 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 6.598178863525391 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 6.595784664154053 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 6.581279754638672 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 6.592929363250732 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 6.599449157714844 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 6.607228755950928 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 6.60359001159668 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 6.591875076293945 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 6.5998334884643555 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 6.584235191345215 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 6.582729339599609 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 6.591054916381836 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 6.602621555328369 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 6.607583045959473 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 6.578174591064453 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 6.580382347106934 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 6.600681781768799 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 6.601696968078613 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 6.586922645568848 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 6.600650787353516 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 6.599786281585693 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 6.593698024749756 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 6.588168144226074 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 6.5824480056762695 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 6.58252477645874 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 6.589319705963135 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 6.601154327392578 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 6.602333068847656 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 6.596490859985352 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 6.599719524383545 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 6.591432571411133 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 6.603636264801025 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 6.603952407836914 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 6.6040825843811035 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 6.601742267608643 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 6.602610111236572 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 6.602569103240967 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 6.585309982299805 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 6.603806495666504 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 6.587503910064697 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 6.593698024749756 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 6.5925798416137695 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 6.5892462730407715 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 6.591400623321533 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 6.60642671585083 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 6.597780704498291 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 6.601413726806641 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 6.59487247467041 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 6.600255012512207 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 6.594669818878174 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 6.588927268981934 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 6.597878932952881 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 6.606900215148926 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 6.590006351470947 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 6.594246864318848 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 6.596985340118408 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 6.601166725158691 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 6.593268871307373 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 6.598250865936279 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 6.605244159698486 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 6.604512691497803 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 6.587610244750977 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 6.601518630981445 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 6.6021952629089355 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 6.605716705322266 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 6.603775501251221 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 6.607298374176025 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 6.6122212409973145 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 6.593820571899414 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 6.599548816680908 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 6.596066474914551 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 6.593171119689941 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 6.599617004394531 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 6.605527400970459 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 6.602389812469482 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 6.598952293395996 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 6.602376937866211 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 6.606771945953369 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 6.602109432220459 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 6.592378616333008 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 6.590651512145996 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 6.6001057624816895 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 6.598791122436523 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 6.604039669036865 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 6.604513168334961 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 6.603310585021973 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 6.588095188140869 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 6.601797580718994 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 6.600030422210693 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 6.606400489807129 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 6.600368499755859 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 6.597190856933594 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 6.589382648468018 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 6.587274074554443 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 6.596391677856445 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 6.59694242477417 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 6.592281341552734 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 6.59852933883667 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 6.57909631729126 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 6.581469535827637 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 6.600376129150391 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 6.596221923828125 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 6.605307579040527 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 6.600579738616943 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 6.60133171081543 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 6.605119705200195 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 6.60227108001709 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 6.598724365234375 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 6.606266975402832 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 6.598618507385254 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 6.598852157592773 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 6.601508140563965 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 6.598553657531738 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 6.601412296295166 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 6.59518575668335 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 6.594697952270508 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 6.597851276397705 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 6.6041178703308105 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 6.600735187530518 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 6.601080894470215 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 6.60016393661499 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 6.5956621170043945 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 6.600144863128662 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 6.597470760345459 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 6.6035685539245605 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 6.603593826293945 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 6.605932712554932 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 6.599174976348877 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 6.595711708068848 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 6.600107669830322 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 6.5986552238464355 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 6.601630210876465 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 6.602270603179932 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 6.6023478507995605 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 6.600612163543701 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 6.587116241455078 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 6.594443321228027 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 6.597299575805664 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 6.605952262878418 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 6.60903263092041 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 6.610475540161133 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 6.610025405883789 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 6.594229221343994 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 6.595969200134277 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 6.597070693969727 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 6.599231719970703 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 6.601463794708252 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 6.602692127227783 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 6.603835105895996 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 6.584806442260742 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 6.598879814147949 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 6.592294216156006 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 6.599766254425049 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 6.602532386779785 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 6.606451988220215 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 6.607882022857666 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 6.596158981323242 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 6.59637451171875 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 6.607106685638428 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 6.607349872589111 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 6.609874248504639 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 6.612264156341553 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 6.604864597320557 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 6.598066806793213 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 6.592206954956055 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 6.596440315246582 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 6.604358196258545 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 6.605189800262451 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 6.613621234893799 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 6.606575965881348 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 6.6003241539001465 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 6.607431888580322 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 6.603500843048096 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 6.61030387878418 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 6.605193614959717 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 6.612126350402832 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 6.603789806365967 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 6.59227991104126 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 6.593422889709473 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 6.596105575561523 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 6.604356288909912 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 6.607855796813965 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 6.598106384277344 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 6.600514888763428 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 6.605868339538574 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 6.606652736663818 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 6.601559162139893 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 6.599547386169434 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 6.603506088256836 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 6.5987725257873535 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 6.601705074310303 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 6.595846176147461 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 6.605261325836182 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 6.6010050773620605 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 6.605607986450195 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 6.607065200805664 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 6.599732875823975 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 6.601710319519043 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 6.601534843444824 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 6.6002373695373535 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 6.601970672607422 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 6.599880218505859 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 6.599278926849365 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 6.596271514892578 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 6.596656799316406 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 6.596795558929443 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 6.602492809295654 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 6.600967884063721 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 6.599152565002441 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 6.597898483276367 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 6.594899654388428 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 6.597871780395508 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 6.596582412719727 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 6.601064205169678 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 6.595006465911865 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 6.603090286254883 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 6.601056098937988 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 6.60007381439209 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 6.606298923492432 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 6.59986686706543 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 6.601884365081787 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 6.59735631942749 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 6.601327896118164 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 6.606303691864014 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 6.604212760925293 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 6.601311206817627 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 6.597829341888428 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 6.6004838943481445 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 6.598099231719971 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 6.608369827270508 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 6.606858730316162 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 6.600128650665283 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 6.6029767990112305 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 6.6003546714782715 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 6.59958028793335 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 6.593913555145264 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 6.604043006896973 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 6.604269027709961 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 6.606225967407227 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 6.606950283050537 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 6.595943450927734 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 6.599369525909424 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 6.598005294799805 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 6.603532791137695 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 6.607446193695068 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 6.605018138885498 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 6.60512638092041 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 6.598582744598389 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 6.5997185707092285 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 6.5999531745910645 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 6.6023406982421875 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 6.602704048156738 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 6.601614952087402 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 6.599346160888672 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 6.5980119705200195 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 6.596429347991943 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 6.596161365509033 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 6.605160236358643 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 6.611623764038086 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 6.610204219818115 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 6.608973026275635 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 6.599440574645996 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 6.597598075866699 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 6.597370624542236 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 6.605222225189209 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 6.604358196258545 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 6.602878570556641 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 6.605030536651611 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 6.5970940589904785 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 6.600498676300049 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 6.596414566040039 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 6.6046833992004395 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 6.6062541007995605 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 6.605898857116699 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 6.607046604156494 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 6.597235679626465 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 6.599125862121582 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 6.597239017486572 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 6.603255748748779 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 6.604758262634277 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 6.60517692565918 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 6.607698440551758 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 6.5967912673950195 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 6.599171161651611 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 6.596045017242432 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 6.606232643127441 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 6.608950138092041 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 6.60835599899292 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 6.607090950012207 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 6.6011457443237305 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 6.600656509399414 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 6.596714019775391 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 6.604354381561279 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 6.605943202972412 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 6.6068572998046875 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 6.608158111572266 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 6.598452568054199 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 6.5994553565979 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 6.598047733306885 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 6.603742599487305 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 6.6061906814575195 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 6.606211185455322 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 6.604738712310791 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 6.5995354652404785 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 6.599943161010742 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 6.5973687171936035 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 6.604788780212402 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 6.605924606323242 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 6.60308837890625 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 6.603743553161621 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 6.598557472229004 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 6.598172664642334 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 6.6009602546691895 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 6.604080677032471 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 6.605562210083008 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 6.605836391448975 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 6.603350639343262 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 6.598554611206055 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 6.6019134521484375 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 6.597433090209961 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 6.606156349182129 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 6.606710433959961 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 6.607789516448975 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 6.6072096824646 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 6.602711200714111 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 6.600829601287842 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 6.599113941192627 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 6.6030168533325195 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 6.602009296417236 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 6.60439920425415 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 6.603659629821777 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 6.598233699798584 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 6.5990753173828125 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 6.597658634185791 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 6.6041483879089355 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 6.604606628417969 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 6.604905605316162 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 6.603374481201172 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 6.599079132080078 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 6.5978217124938965 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 6.6002726554870605 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 6.604966640472412 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 6.604852199554443 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 6.6043243408203125 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 6.604616641998291 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 6.599572658538818 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 6.601479530334473 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 6.600064754486084 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 6.604489803314209 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 6.604806423187256 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 6.606635093688965 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 6.60684871673584 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 6.602787017822266 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 6.6002655029296875 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 6.60007905960083 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 6.6053385734558105 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 6.604893207550049 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 6.605632305145264 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 6.604212760925293 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 6.600472450256348 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 6.59916353225708 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 6.600151538848877 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 6.6047563552856445 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 6.605164051055908 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 6.60540246963501 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 6.605785846710205 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 6.597067356109619 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 6.599972248077393 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 6.600137233734131 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 6.604395389556885 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 6.6043524742126465 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 6.604781150817871 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 6.605875492095947 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 6.599791049957275 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 6.601497173309326 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 6.599152565002441 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 6.603846073150635 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 6.604429721832275 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 6.6054158210754395 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 6.604636192321777 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 6.599225997924805 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 6.6016764640808105 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 6.600290298461914 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 6.604108810424805 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 6.604352951049805 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 6.604194641113281 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 6.605434417724609 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 6.599965572357178 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 6.600320339202881 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 6.600098133087158 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 6.603488445281982 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 6.604409694671631 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 6.605992317199707 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 6.605588912963867 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 6.5988593101501465 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 6.599915504455566 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 6.602593898773193 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 6.603749752044678 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 6.603925704956055 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 6.603025913238525 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 6.603877544403076 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 6.600656509399414 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 6.601109981536865 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 6.597596168518066 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 6.605644226074219 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 6.605431079864502 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 6.603664875030518 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 6.604753017425537 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 6.60113525390625 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 6.6016106605529785 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 6.600804328918457 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 6.604508876800537 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 6.606046676635742 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 6.603638648986816 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 6.607027053833008 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 6.5996527671813965 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 6.599191665649414 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 6.600780487060547 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 6.603200435638428 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 6.603689670562744 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 6.599296569824219 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 6.601716041564941 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 6.599742889404297 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 6.598677158355713 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 6.601217746734619 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 6.6124982833862305 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 6.612306118011475 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 6.606039524078369 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 6.602809906005859 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 6.601545333862305 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 6.604678630828857 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 6.602466583251953 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 6.603470802307129 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 6.603786945343018 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 6.604178428649902 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 6.602931022644043 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 6.599786281585693 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 6.602194309234619 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 6.6041646003723145 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 6.619690418243408 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 6.620758533477783 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 6.615204811096191 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 6.61479377746582 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 6.595831871032715 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 6.594951629638672 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 6.601993083953857 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 6.605774402618408 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 6.605727672576904 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 6.606330871582031 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 6.608585357666016 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 6.6034440994262695 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 6.605523109436035 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 6.60240364074707 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 6.633901596069336 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 6.626533508300781 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 6.61317253112793 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 6.613747596740723 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 6.583765506744385 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 6.603389263153076 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 6.60793924331665 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 6.605195999145508 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 6.6029372215271 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 6.603430271148682 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 6.605612277984619 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 6.606897830963135 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 6.604814052581787 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 6.606776714324951 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 6.604368686676025 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 6.604959011077881 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 6.617969036102295 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 6.60932731628418 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 6.589051246643066 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 6.588523864746094 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 6.589207649230957 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 6.603640079498291 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 6.6061506271362305 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 6.60843563079834 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 6.607334136962891 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 6.621618747711182 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 6.6069841384887695 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 6.545148849487305 - } - ] - }, - { - "bit": 3, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 4.370658874511719 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 4.3709917068481445 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 4.372006416320801 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 4.370563507080078 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 4.37232780456543 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 4.373668670654297 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 4.3709635734558105 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 4.367496013641357 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 4.368441581726074 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 4.367706775665283 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 4.365130424499512 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 4.367547988891602 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 4.369000434875488 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 4.367656230926514 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 4.368114948272705 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 4.369100570678711 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 4.367117881774902 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 4.366388320922852 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 4.36487340927124 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 4.368755340576172 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 4.365574836730957 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 4.368518352508545 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 4.369019508361816 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 4.369363307952881 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 4.368993282318115 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 4.3654866218566895 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 4.367645263671875 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 4.368037223815918 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 4.369693279266357 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 4.369786262512207 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 4.369330883026123 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 4.36944580078125 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 4.366352081298828 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 4.368671894073486 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 4.368444442749023 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 4.371212482452393 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 4.372028827667236 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 4.370617389678955 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 4.370952606201172 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 4.366947174072266 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 4.368783473968506 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 4.369964122772217 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 4.3701066970825195 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 4.370917320251465 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 4.37069034576416 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 4.369626522064209 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 4.367837429046631 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 4.3685102462768555 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 4.367172718048096 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 4.371077060699463 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 4.370008945465088 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 4.370212078094482 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 4.370441436767578 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 4.369170188903809 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 4.366943836212158 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 4.369058132171631 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 4.370612621307373 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 4.371523857116699 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 4.369246959686279 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 4.367514133453369 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 4.367551326751709 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 4.366318702697754 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 4.368446350097656 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 4.370789051055908 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 4.3708600997924805 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 4.370893955230713 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 4.370158672332764 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 4.367159843444824 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 4.370361804962158 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 4.367939472198486 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 4.37029504776001 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 4.370350360870361 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 4.369771480560303 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 4.369573593139648 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 4.366819381713867 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 4.3677144050598145 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 4.368299961090088 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 4.37019157409668 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 4.371087074279785 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 4.370404243469238 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 4.3695831298828125 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 4.366743564605713 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 4.369704723358154 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 4.36887264251709 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 4.370004653930664 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 4.370041847229004 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 4.370377063751221 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 4.371026992797852 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 4.36906623840332 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 4.370051383972168 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 4.369397163391113 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 4.370617389678955 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 4.3704142570495605 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 4.368880748748779 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 4.36940336227417 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 4.369184970855713 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 4.369528770446777 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 4.3685126304626465 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 4.369605541229248 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 4.369452476501465 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 4.370082855224609 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 4.369543075561523 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 4.36824893951416 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 4.368817329406738 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 4.370004177093506 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 4.371281147003174 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 4.3708906173706055 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 4.370126724243164 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 4.369634628295898 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 4.368972301483154 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 4.37039041519165 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 4.369751453399658 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 4.370290279388428 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 4.370081424713135 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 4.370199680328369 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 4.369494438171387 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 4.36939001083374 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 4.369959354400635 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 4.36924409866333 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 4.369386196136475 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 4.3701677322387695 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 4.368776798248291 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 4.370244979858398 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 4.370024681091309 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 4.371572971343994 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 4.369118690490723 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 4.369749546051025 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 4.36985969543457 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 4.36874532699585 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 4.36946439743042 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 4.369516372680664 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 4.369718551635742 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 4.370192527770996 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 4.37045431137085 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 4.370177268981934 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 4.3698554039001465 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 4.36964750289917 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 4.368263244628906 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 4.370279312133789 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 4.370323657989502 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 4.3698954582214355 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 4.370196342468262 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 4.36991024017334 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 4.369865417480469 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 4.369717121124268 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 4.368814468383789 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 4.368879318237305 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 4.369174480438232 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 4.369974613189697 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 4.369632720947266 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 4.369807720184326 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 4.369532585144043 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 4.369607448577881 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 4.370362281799316 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 4.370495319366455 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 4.370677947998047 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 4.370682716369629 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 4.370794296264648 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 4.369177341461182 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 4.3691606521606445 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 4.369308948516846 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 4.369301795959473 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 4.370370388031006 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 4.370275020599365 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 4.369775772094727 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 4.369058132171631 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 4.369899272918701 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 4.369477272033691 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 4.3701043128967285 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 4.370334148406982 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 4.370373725891113 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 4.370621681213379 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 4.368663311004639 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 4.369245529174805 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 4.370733261108398 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 4.369558811187744 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 4.37009859085083 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 4.3681817054748535 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 4.367650032043457 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 4.369901657104492 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 4.369140148162842 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 4.370650768280029 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 4.37069034576416 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 4.370615482330322 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 4.370699405670166 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 4.370719909667969 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 4.369001865386963 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 4.369406700134277 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 4.370663166046143 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 4.369678497314453 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 4.369998455047607 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 4.37005615234375 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 4.369414806365967 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 4.369797229766846 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 4.369734764099121 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 4.369627952575684 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 4.370367527008057 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 4.370331764221191 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 4.370481491088867 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 4.369726657867432 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 4.369414806365967 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 4.369966506958008 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 4.368894577026367 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 4.370353698730469 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 4.3705620765686035 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 4.370404243469238 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 4.369851589202881 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 4.369944095611572 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 4.36950159072876 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 4.37017297744751 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 4.370259761810303 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 4.370347499847412 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 4.370332717895508 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 4.370377540588379 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 4.369397163391113 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 4.369746685028076 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 4.368403911590576 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 4.369879722595215 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 4.370453834533691 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 4.3704681396484375 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 4.3705668449401855 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 4.369260787963867 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 4.3699421882629395 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 4.368878364562988 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 4.370258331298828 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 4.370457172393799 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 4.3696722984313965 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 4.3701300621032715 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 4.367971420288086 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 4.369325637817383 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 4.36854887008667 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 4.370546817779541 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 4.370273113250732 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 4.369988918304443 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 4.369546413421631 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 4.367751598358154 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 4.369838237762451 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 4.369894027709961 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 4.370205879211426 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 4.370344638824463 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 4.369682312011719 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 4.369753360748291 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 4.369647026062012 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 4.369004726409912 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 4.370223522186279 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 4.370478630065918 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 4.370723724365234 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 4.36993408203125 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 4.370380878448486 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 4.369374752044678 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 4.370112419128418 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 4.369381427764893 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 4.370210647583008 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 4.370704174041748 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 4.370889186859131 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 4.370656967163086 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 4.36891508102417 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 4.370216369628906 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 4.36879825592041 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 4.370626926422119 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 4.3706440925598145 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 4.370516300201416 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 4.370355129241943 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 4.369748592376709 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 4.370649814605713 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 4.37078332901001 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 4.370316982269287 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 4.370417594909668 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 4.369919776916504 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 4.3699798583984375 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 4.3686203956604 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 4.369569301605225 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 4.3697309494018555 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 4.369585990905762 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 4.370404243469238 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 4.370459079742432 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 4.369085311889648 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 4.369277000427246 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 4.370120525360107 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 4.3698225021362305 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 4.369800567626953 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 4.369987964630127 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 4.369215965270996 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 4.369477272033691 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 4.369507789611816 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 4.3702311515808105 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 4.369599342346191 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 4.369622230529785 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 4.369960784912109 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 4.369366645812988 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 4.370115280151367 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 4.368876934051514 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 4.369987964630127 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 4.3692946434021 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 4.371050834655762 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 4.370308876037598 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 4.370563507080078 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 4.370288372039795 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 4.368695259094238 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 4.370031833648682 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 4.37069845199585 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 4.369816780090332 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 4.370055198669434 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 4.369913101196289 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 4.369883060455322 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 4.369180679321289 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 4.3699164390563965 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 4.369873523712158 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 4.370911598205566 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 4.370357036590576 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 4.3701300621032715 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 4.369647979736328 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 4.36823844909668 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 4.368480682373047 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 4.368778228759766 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 4.370745658874512 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 4.370642185211182 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 4.370589256286621 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 4.370792388916016 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 4.369303226470947 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 4.369446754455566 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 4.368327617645264 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 4.370411396026611 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 4.370616436004639 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 4.370595932006836 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 4.370485305786133 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 4.369943141937256 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 4.369462490081787 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 4.370121002197266 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 4.3701372146606445 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 4.370247840881348 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 4.370376110076904 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 4.369966506958008 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 4.3690266609191895 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 4.3697829246521 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 4.369048595428467 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 4.370481014251709 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 4.370688438415527 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 4.370548725128174 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 4.370810508728027 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 4.368679523468018 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 4.369833469390869 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 4.3692827224731445 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 4.370330333709717 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 4.370555877685547 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 4.37021541595459 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 4.37031364440918 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 4.368698596954346 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 4.369692325592041 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 4.3687238693237305 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 4.370598793029785 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 4.3705058097839355 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 4.370670795440674 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 4.370545387268066 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 4.369062900543213 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 4.369428634643555 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 4.369572639465332 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 4.370123386383057 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 4.370323657989502 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 4.37005615234375 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 4.370494842529297 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 4.368701934814453 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 4.370166301727295 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 4.368327617645264 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 4.3705291748046875 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 4.370993614196777 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 4.370922088623047 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 4.370280742645264 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 4.368600845336914 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 4.369245529174805 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 4.368486404418945 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 4.3705644607543945 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 4.370523452758789 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 4.3705854415893555 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 4.370707035064697 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 4.368817329406738 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 4.369629383087158 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 4.36944055557251 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 4.370484352111816 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 4.370792388916016 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 4.370545864105225 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 4.370151519775391 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 4.369105815887451 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 4.369824409484863 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 4.36972188949585 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 4.370765686035156 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 4.370667457580566 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 4.370337009429932 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 4.37004280090332 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 4.369317054748535 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 4.369020462036133 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 4.3690080642700195 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 4.370368957519531 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 4.370558261871338 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 4.370368957519531 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 4.370271682739258 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 4.367918014526367 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 4.369907855987549 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 4.368680953979492 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 4.370500087738037 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 4.37069845199585 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 4.370538234710693 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 4.370504379272461 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 4.368698596954346 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 4.370347023010254 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 4.369478702545166 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 4.370362281799316 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 4.369499683380127 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 4.370457172393799 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 4.370245933532715 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 4.369790554046631 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 4.369180679321289 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 4.3686676025390625 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 4.370546817779541 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 4.3705244064331055 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 4.370411396026611 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 4.370611190795898 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 4.368659973144531 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 4.370439529418945 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 4.368627548217773 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 4.370355129241943 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 4.3705339431762695 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 4.370579719543457 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 4.370670318603516 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 4.368842124938965 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 4.368569374084473 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 4.368898391723633 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 4.370463848114014 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 4.370635986328125 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 4.370703220367432 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 4.370877265930176 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 4.369894027709961 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 4.369246482849121 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 4.368661880493164 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 4.370605945587158 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 4.370710372924805 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 4.370368957519531 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 4.370603084564209 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 4.368288993835449 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 4.368137359619141 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 4.369358539581299 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 4.370471000671387 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 4.370543479919434 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 4.3706560134887695 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 4.3705315589904785 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 4.367757320404053 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 4.369256019592285 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 4.368664741516113 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 4.370599746704102 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 4.370527267456055 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 4.370788097381592 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 4.370481967926025 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 4.3685994148254395 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 4.369345188140869 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 4.3690643310546875 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 4.370444297790527 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 4.370588302612305 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 4.370532989501953 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 4.3704423904418945 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 4.368338108062744 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 4.369311332702637 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 4.3697028160095215 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 4.370581150054932 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 4.370548248291016 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 4.370380878448486 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 4.370533466339111 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 4.368878364562988 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 4.369256019592285 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 4.369171142578125 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 4.370511531829834 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 4.370306015014648 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 4.372114658355713 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 4.37081241607666 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 4.368305683135986 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 4.369886875152588 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 4.369573593139648 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 4.370284080505371 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 4.370397567749023 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 4.370340824127197 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 4.3701066970825195 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 4.3685431480407715 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 4.369591236114502 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 4.369800090789795 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 4.370703220367432 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 4.370458126068115 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 4.3702239990234375 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 4.370509147644043 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 4.369507789611816 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 4.36972713470459 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 4.3690361976623535 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 4.370020389556885 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 4.369931221008301 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 4.370157241821289 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 4.370376110076904 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 4.3678412437438965 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 4.369663238525391 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 4.370247840881348 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 4.370338439941406 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 4.370195388793945 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 4.370934009552002 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 4.370158672332764 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 4.368406772613525 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 4.369710922241211 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 4.3704047203063965 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 4.37039852142334 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 4.370025157928467 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 4.370766639709473 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 4.370181083679199 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 4.368619918823242 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 4.369656085968018 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 4.369997024536133 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 4.370120525360107 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 4.370104789733887 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 4.370178699493408 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 4.370549201965332 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 4.369039058685303 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 4.369682788848877 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 4.370556354522705 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 4.3708176612854 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 4.371338844299316 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 4.3713698387146 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 4.370495319366455 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 4.367344379425049 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 4.370899677276611 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 4.369388103485107 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 4.3706560134887695 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 4.370896816253662 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 4.370218753814697 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 4.370737552642822 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 4.366901397705078 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 4.369959831237793 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 4.370751857757568 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 4.372854232788086 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 4.37371826171875 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 4.372132301330566 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 4.371866226196289 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 4.368535995483398 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 4.370369911193848 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 4.370703220367432 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 4.370266437530518 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 4.370609283447266 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 4.3709564208984375 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 4.370542526245117 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 4.367404937744141 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 4.371592998504639 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 4.369471549987793 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 4.370596408843994 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 4.370932102203369 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 4.371057510375977 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 4.370301723480225 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 4.365617752075195 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 4.367737293243408 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 4.370293617248535 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 4.370628833770752 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 4.3704962730407715 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 4.370964050292969 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 4.370872974395752 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 4.37193489074707 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 4.3698039054870605 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 4.364621639251709 - } - ] - }, - { - "bit": 4, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3.989259958267212 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3.989586591720581 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3.989457607269287 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 3.989313840866089 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3.9897987842559814 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3.9898786544799805 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 3.990248203277588 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3.9888205528259277 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3.9888482093811035 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3.988792657852173 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3.988403797149658 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3.9887535572052 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3.9891655445098877 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3.9890999794006348 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3.988828659057617 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3.9891767501831055 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3.9891536235809326 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3.9891157150268555 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3.9883594512939453 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3.9891726970672607 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3.9888460636138916 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3.9889020919799805 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3.9890296459198 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 3.9892048835754395 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3.9892704486846924 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3.988804340362549 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3.9889495372772217 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3.988839864730835 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3.989001750946045 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3.9892921447753906 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3.9890191555023193 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3.989163875579834 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3.9881536960601807 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3.9889028072357178 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3.988525152206421 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3.989436149597168 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3.98948335647583 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3.989192247390747 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3.9888899326324463 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3.9885690212249756 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3.989027738571167 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3.9890570640563965 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3.9894745349884033 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3.9904227256774902 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 3.9890286922454834 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3.9892079830169678 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3.988530397415161 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3.9888336658477783 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3.9892683029174805 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3.98945951461792 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3.9891886711120605 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3.9894843101501465 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3.9893927574157715 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3.9892241954803467 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3.9896702766418457 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3.988649368286133 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3.9897477626800537 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3.9896512031555176 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3.9891316890716553 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3.9886972904205322 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3.9881975650787354 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3.988278865814209 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3.988563060760498 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3.9895379543304443 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3.9894497394561768 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3.9892513751983643 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3.989354372024536 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3.988262891769409 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3.9891200065612793 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3.989325523376465 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3.9892842769622803 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3.9892308712005615 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3.9894282817840576 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3.9892523288726807 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3.987931251525879 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3.9885494709014893 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3.9885199069976807 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3.9891483783721924 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3.989225149154663 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3.989548921585083 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3.988978385925293 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3.989313840866089 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3.9890308380126953 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3.9890964031219482 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3.9892144203186035 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3.9892191886901855 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3.9894723892211914 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3.9888992309570312 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3.9884533882141113 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3.988980293273926 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3.988579750061035 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3.9894180297851562 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3.9894254207611084 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3.9893949031829834 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3.988919258117676 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3.988879680633545 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3.989042043685913 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3.9887733459472656 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3.9894254207611084 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3.9891810417175293 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3.989112377166748 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3.9888577461242676 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3.9885380268096924 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3.9889726638793945 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3.9891836643218994 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3.9896981716156006 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3.9893062114715576 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3.989243745803833 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3.9898037910461426 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3.988999128341675 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3.989009380340576 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3.9892008304595947 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3.9893577098846436 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3.989319086074829 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3.9888699054718018 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3.9891715049743652 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3.9891586303710938 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3.9890296459198 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3.9885408878326416 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3.9893174171447754 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3.9891855716705322 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3.9888226985931396 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3.988914728164673 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3.9891855716705322 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3.989671468734741 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3.9894156455993652 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3.9892466068267822 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3.9891998767852783 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3.988955020904541 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3.9889097213745117 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3.9881837368011475 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3.988870143890381 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3.988237142562866 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3.989407777786255 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3.9892232418060303 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3.988959312438965 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3.9897563457489014 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3.988868474960327 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3.9888534545898438 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3.988537073135376 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3.989206552505493 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3.989356756210327 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3.9894490242004395 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3.9891295433044434 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3.9892303943634033 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3.989528179168701 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3.9892919063568115 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3.989149332046509 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3.9892704486846924 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3.9888741970062256 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3.98921799659729 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3.988981246948242 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3.9894840717315674 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3.98880672454834 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3.989325523376465 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3.9894766807556152 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3.989553928375244 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3.989331007003784 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3.9888322353363037 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3.9893970489501953 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3.989107608795166 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3.989248275756836 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3.9893062114715576 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3.989201784133911 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3.9893617630004883 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3.9887330532073975 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3.988969326019287 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3.9886586666107178 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3.9891247749328613 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3.9890761375427246 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3.989189624786377 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3.9892144203186035 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3.9881975650787354 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3.9889137744903564 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3.9891319274902344 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3.9891629219055176 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3.9892027378082275 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3.9889299869537354 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3.988969326019287 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3.9887330532073975 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3.9891817569732666 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3.988999128341675 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3.9893929958343506 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3.989374876022339 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3.9892189502716064 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3.9891698360443115 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3.989150285720825 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3.988983631134033 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3.9890239238739014 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3.9891390800476074 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3.989150285720825 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3.9892122745513916 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3.988948345184326 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3.9888322353363037 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3.9887423515319824 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3.9891672134399414 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3.9893393516540527 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3.9892256259918213 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3.989337921142578 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3.9892892837524414 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3.989330291748047 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3.989302635192871 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3.989403009414673 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3.9891579151153564 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3.9892680644989014 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3.989086866378784 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3.9888813495635986 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3.9889028072357178 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3.989297866821289 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3.988936424255371 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3.9893620014190674 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3.989352226257324 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3.9891886711120605 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3.9892687797546387 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3.9893035888671875 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3.9888899326324463 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3.9887938499450684 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3.989150285720825 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3.9891417026519775 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3.9893686771392822 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3.9892961978912354 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3.988906145095825 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3.9887096881866455 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3.9892475605010986 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3.989457607269287 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3.989461660385132 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3.9895946979522705 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3.989490509033203 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3.9885165691375732 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3.9887421131134033 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3.989222526550293 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3.989403486251831 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3.9892351627349854 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3.9890494346618652 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3.9894766807556152 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3.988926887512207 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3.988981246948242 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3.989095687866211 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3.9890990257263184 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3.9892289638519287 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3.988813877105713 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3.9888675212860107 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3.9890201091766357 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3.989264488220215 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3.9891629219055176 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3.9892687797546387 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3.9892332553863525 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3.9892725944519043 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3.9892780780792236 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3.988875150680542 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3.9892826080322266 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3.9892654418945312 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3.9891510009765625 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3.9892423152923584 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3.9888503551483154 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3.989189624786377 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3.9888744354248047 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3.988682746887207 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3.988732099533081 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3.989354372024536 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3.989346981048584 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3.989440441131592 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3.9893927574157715 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3.988666534423828 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3.9890897274017334 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3.989128589630127 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3.9891953468322754 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3.9893593788146973 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3.9888129234313965 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3.9893574714660645 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3.9888243675231934 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3.9891157150268555 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3.9889512062072754 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3.988999843597412 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3.989187717437744 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3.9890007972717285 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3.989269495010376 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3.9888978004455566 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3.9890239238739014 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3.989375591278076 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3.9889957904815674 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3.989133834838867 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3.9887492656707764 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3.9885671138763428 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3.9892354011535645 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3.9889469146728516 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3.989124298095703 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3.9891810417175293 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3.988983631134033 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3.988938570022583 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3.989008665084839 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3.9890785217285156 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3.9894680976867676 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3.98892879486084 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3.989581346511841 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3.9892544746398926 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3.9894206523895264 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3.988901138305664 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3.9890878200531006 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3.9896774291992188 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3.9888594150543213 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3.989075183868408 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3.98915433883667 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3.9888830184936523 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3.9889955520629883 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3.98901629447937 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3.9893953800201416 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3.9888975620269775 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3.989729642868042 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3.9893274307250977 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3.9893341064453125 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3.9893274307250977 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3.989074945449829 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3.989069700241089 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3.988879442214966 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3.989363670349121 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3.989429235458374 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3.9894800186157227 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3.9893863201141357 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3.988861083984375 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3.988797664642334 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3.9886624813079834 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3.989293098449707 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3.989344835281372 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3.9892749786376953 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3.9892332553863525 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3.9891910552978516 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3.9891397953033447 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3.988936424255371 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3.9892332553863525 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3.9892208576202393 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3.9889087677001953 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3.989064931869507 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3.9886155128479004 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3.989206075668335 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3.988797664642334 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3.9893131256103516 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3.9894802570343018 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3.989400625228882 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3.989354133605957 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3.9890501499176025 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3.988982677459717 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3.9890353679656982 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3.98925518989563 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3.989328622817993 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3.989320755004883 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3.989302635192871 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3.989107131958008 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3.9888741970062256 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3.989039421081543 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3.9893078804016113 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3.989363670349121 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3.989313840866089 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3.989330291748047 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3.9889299869537354 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3.9892492294311523 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3.9886844158172607 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3.98925518989563 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3.9892446994781494 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3.989201784133911 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3.9889707565307617 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3.9891231060028076 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3.988461971282959 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3.9891316890716553 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3.9892756938934326 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3.9894351959228516 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3.989367961883545 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3.9893722534179688 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3.988734722137451 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3.989142894744873 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3.988715887069702 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3.9892406463623047 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3.9892940521240234 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3.9893600940704346 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3.989360809326172 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3.9891810417175293 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3.9887373447418213 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3.9891562461853027 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3.9893147945404053 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3.989366292953491 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3.9893455505371094 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3.9892489910125732 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3.988682985305786 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3.9891204833984375 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3.98909330368042 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3.9894819259643555 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3.9893782138824463 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3.9893758296966553 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3.9894521236419678 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3.9892423152923584 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3.989129066467285 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3.9885218143463135 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3.9893016815185547 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3.9893746376037598 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3.989356756210327 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3.98934268951416 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3.9886624813079834 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3.9887986183166504 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3.9892826080322266 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3.989372968673706 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3.98942232131958 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3.989434242248535 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3.989335060119629 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3.989321708679199 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3.988795757293701 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3.989323139190674 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3.9892122745513916 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3.989243745803833 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3.989248275756836 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3.989243745803833 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3.9882888793945312 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3.9884207248687744 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3.988530397415161 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3.9893617630004883 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3.989356756210327 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3.9893007278442383 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3.989471197128296 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3.988717555999756 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3.9889183044433594 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3.9884607791900635 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3.9893126487731934 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3.9893693923950195 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3.989370346069336 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3.989490032196045 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3.9890732765197754 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3.9888288974761963 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3.988673210144043 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3.989349126815796 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3.989354133605957 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3.989321231842041 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3.9893245697021484 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3.98811936378479 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3.9893627166748047 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3.9885494709014893 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3.9893686771392822 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3.9893672466278076 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3.989360809326172 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3.989272356033325 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3.9892351627349854 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3.989089012145996 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3.989013671875 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3.9893293380737305 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3.9893362522125244 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3.9893863201141357 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3.989257574081421 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3.9890921115875244 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3.988661527633667 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3.989065408706665 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3.9893274307250977 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3.989316940307617 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3.989312171936035 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3.9893853664398193 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3.9880311489105225 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3.9887709617614746 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3.9890472888946533 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3.9893574714660645 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3.989388942718506 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3.9894111156463623 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3.989398717880249 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3.988504648208618 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3.988600730895996 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3.988816499710083 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3.9893383979797363 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3.9893414974212646 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3.9893970489501953 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3.989328384399414 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3.9888129234313965 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3.988917350769043 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3.9889075756073 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3.989337205886841 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3.9893174171447754 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3.9898808002471924 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3.9893643856048584 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3.9892351627349854 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3.9893455505371094 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3.988438129425049 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3.989210844039917 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3.989331007003784 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3.989150285720825 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3.9894258975982666 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3.9890072345733643 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3.988896608352661 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3.9886176586151123 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3.989475727081299 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3.9894845485687256 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3.989161968231201 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3.989293098449707 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3.988750457763672 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3.988602638244629 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3.9887380599975586 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3.98919415473938 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3.98934268951416 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3.9889636039733887 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3.989222526550293 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3.9883649349212646 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3.989312171936035 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3.988905191421509 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3.989315986633301 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3.989269733428955 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3.9892523288726807 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3.9893949031829834 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3.988466739654541 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3.9892351627349854 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3.9892759323120117 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3.989077091217041 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3.989283561706543 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3.9891998767852783 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3.989274024963379 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3.9886484146118164 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3.9887261390686035 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3.9891810417175293 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3.989163875579834 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3.989224672317505 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3.989614248275757 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3.9892940521240234 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3.987661361694336 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3.9894137382507324 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3.989142894744873 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3.989269495010376 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3.9892876148223877 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3.989215612411499 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3.9893929958343506 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3.9886398315429688 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3.9891321659088135 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3.9897193908691406 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3.9893102645874023 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3.989388942718506 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3.989494800567627 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3.9892327785491943 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3.98606014251709 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3.9885873794555664 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3.9890239238739014 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3.9897992610931396 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3.989866018295288 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3.9900646209716797 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3.98954701423645 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3.9875223636627197 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3.9895410537719727 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3.9891552925109863 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3.989344596862793 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3.9893686771392822 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3.989469051361084 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3.98952317237854 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3.9880974292755127 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3.989020586013794 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3.9886622428894043 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3.9893083572387695 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3.989287853240967 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3.9888699054718018 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3.989025831222534 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3.9881908893585205 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3.9891300201416016 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3.9883642196655273 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3.989490509033203 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3.989255666732788 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3.989182710647583 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3.9893696308135986 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3.989755868911743 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3.9892501831054688 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3.988036870956421 - } - ] - }, - { - "bit": 5, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3.8670127391815186 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3.8670167922973633 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3.867168664932251 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 3.8669652938842773 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3.8670926094055176 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3.867605686187744 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 3.867151975631714 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3.8668947219848633 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3.8668062686920166 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3.8667726516723633 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3.8669021129608154 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3.8667140007019043 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3.8669979572296143 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3.86676287651062 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3.866762399673462 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3.8667709827423096 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3.866797924041748 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3.8667521476745605 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3.8666634559631348 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3.866442918777466 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3.8667874336242676 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3.867043972015381 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3.8668901920318604 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 3.8672027587890625 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3.8670194149017334 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3.8665802478790283 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3.8666906356811523 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3.866523265838623 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3.866868734359741 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3.8672447204589844 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3.867022752761841 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3.866962194442749 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3.867201805114746 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3.8669803142547607 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3.8668248653411865 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3.866929292678833 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3.8671927452087402 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3.8667633533477783 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3.8669729232788086 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3.8667445182800293 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3.8669207096099854 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3.866957902908325 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3.867109537124634 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3.867114305496216 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 3.867183208465576 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3.8667140007019043 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3.8670449256896973 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3.866605281829834 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3.8670549392700195 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3.8671059608459473 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3.8669729232788086 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3.8672289848327637 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3.8668394088745117 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3.8669040203094482 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3.867095708847046 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3.8664863109588623 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3.8671257495880127 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3.8671748638153076 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3.867020606994629 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3.8672966957092285 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3.8670146465301514 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3.866345167160034 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3.8670074939727783 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3.8671298027038574 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3.8669424057006836 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3.8672258853912354 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3.8671085834503174 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3.8671653270721436 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3.8668854236602783 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3.8667538166046143 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3.8670730590820312 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3.866971015930176 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3.8669281005859375 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3.8668081760406494 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3.866734266281128 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3.8670272827148438 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3.8666024208068848 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3.866917371749878 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3.8670432567596436 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3.867030143737793 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3.8669376373291016 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3.8668699264526367 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3.86669921875 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3.8669841289520264 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3.8670156002044678 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3.866981267929077 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3.8672358989715576 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3.866919994354248 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3.8665575981140137 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3.866607427597046 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3.867175579071045 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3.866893768310547 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3.8669395446777344 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3.866971969604492 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3.867070436477661 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3.8668861389160156 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3.867039442062378 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3.867255210876465 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3.866832971572876 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3.866960287094116 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3.8672008514404297 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3.8669052124023438 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3.8666751384735107 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3.866931438446045 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3.866826057434082 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3.8671183586120605 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3.8671491146087646 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3.8671178817749023 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3.867034435272217 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3.866884231567383 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3.866901159286499 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3.8668808937072754 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3.8669114112854004 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3.866931438446045 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3.8670709133148193 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3.866968870162964 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3.8669142723083496 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3.8668315410614014 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3.8671939373016357 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3.8669841289520264 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3.8668625354766846 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3.867145538330078 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3.8670687675476074 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3.8669841289520264 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3.8665831089019775 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3.8670477867126465 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3.866914987564087 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3.8668746948242188 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3.8665854930877686 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3.8667032718658447 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3.8666491508483887 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3.8666250705718994 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3.8666863441467285 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3.866941452026367 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3.867124080657959 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3.867203712463379 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3.867032051086426 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3.8667919635772705 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3.866827964782715 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3.867048740386963 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3.8671011924743652 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3.8669114112854004 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3.8668222427368164 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3.8669943809509277 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3.8669159412384033 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3.8672728538513184 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3.8668999671936035 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3.866879940032959 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3.866971015930176 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3.8670156002044678 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3.8667304515838623 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3.8665876388549805 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3.8667845726013184 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3.867292642593384 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3.8671278953552246 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3.866976022720337 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3.866809129714966 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3.8668365478515625 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3.8670573234558105 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3.866831064224243 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3.8668503761291504 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3.8669519424438477 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3.8669090270996094 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3.8671658039093018 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3.867016553878784 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3.8666462898254395 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3.8669309616088867 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3.8672611713409424 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3.8668148517608643 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3.8669114112854004 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3.8668017387390137 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3.866878032684326 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3.867220401763916 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3.867079734802246 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3.866837501525879 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3.8669886589050293 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3.8669509887695312 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3.8668007850646973 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3.8667352199554443 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3.867130994796753 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3.8670597076416016 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3.8667569160461426 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3.8669676780700684 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3.866913318634033 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3.8670477867126465 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3.8671603202819824 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3.866837501525879 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3.866730213165283 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3.8670408725738525 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3.866727113723755 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3.866922616958618 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3.8668394088745117 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3.8669002056121826 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3.866852283477783 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3.8668315410614014 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3.866859197616577 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3.8670871257781982 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3.866957426071167 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3.866899251937866 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3.866835594177246 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3.866734266281128 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3.8673980236053467 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3.8668112754821777 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3.8670239448547363 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3.866957426071167 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3.8668909072875977 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3.8669309616088867 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3.8669300079345703 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3.866875171661377 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3.866528272628784 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3.8668975830078125 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3.8669240474700928 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3.8669769763946533 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3.8667802810668945 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3.8665931224823 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3.8670718669891357 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3.8667564392089844 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3.8668699264526367 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3.866770029067993 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3.866856098175049 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3.8670687675476074 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3.8668198585510254 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3.8670365810394287 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3.8670175075531006 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3.8670241832733154 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3.867119789123535 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3.867015838623047 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3.867100238800049 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3.866541624069214 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3.866865634918213 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3.8668019771575928 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3.8669464588165283 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3.8670976161956787 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3.8671021461486816 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3.867053270339966 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3.8668465614318848 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3.86690092086792 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3.8669018745422363 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3.866846799850464 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3.8668315410614014 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3.8667237758636475 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3.8666577339172363 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3.867318868637085 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3.8671000003814697 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3.8665852546691895 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3.8668477535247803 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3.867053270339966 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3.8669049739837646 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3.8667731285095215 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3.8669071197509766 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3.86676287651062 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3.866442918777466 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3.8671436309814453 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3.8670554161071777 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3.866931438446045 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3.867128849029541 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3.8666789531707764 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3.86680006980896 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3.866840362548828 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3.866919755935669 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3.8669896125793457 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3.8670027256011963 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3.8669090270996094 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3.8669040203094482 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3.867034912109375 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3.867018461227417 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3.8670291900634766 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3.8669095039367676 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3.867007255554199 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3.8669488430023193 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3.8669371604919434 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3.8666892051696777 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3.866647720336914 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3.8667874336242676 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3.8669538497924805 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3.8667564392089844 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3.8667709827423096 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3.866901159286499 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3.8670008182525635 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3.867310047149658 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3.8668439388275146 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3.866814136505127 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3.8666956424713135 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3.866597890853882 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3.8667941093444824 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3.866978883743286 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3.8668413162231445 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3.86692476272583 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3.8668882846832275 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3.866881847381592 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3.866659641265869 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3.866884469985962 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3.866960287094116 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3.8670127391815186 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3.8669960498809814 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3.8668668270111084 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3.866884469985962 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3.866966724395752 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3.8665950298309326 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3.86700177192688 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3.8667337894439697 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3.866954803466797 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3.866886854171753 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3.8670308589935303 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3.866755962371826 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3.867039442062378 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3.866835594177246 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3.866539478302002 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3.867145538330078 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3.86712384223938 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3.8672690391540527 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3.8670644760131836 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3.8667666912078857 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3.867043972015381 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3.866879940032959 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3.8670523166656494 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3.8670551776885986 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3.867204427719116 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3.8668909072875977 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3.8669538497924805 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3.867194414138794 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3.8666107654571533 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3.8670156002044678 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3.8669216632843018 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3.8668572902679443 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3.866960287094116 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3.866706609725952 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3.8666112422943115 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3.8667783737182617 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3.866912364959717 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3.866852283477783 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3.8669021129608154 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3.8669915199279785 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3.866647720336914 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3.867112636566162 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3.8671019077301025 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3.8669281005859375 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3.866964817047119 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3.866971015930176 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3.866931200027466 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3.866687297821045 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3.867095470428467 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3.8669960498809814 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3.8669075965881348 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3.866990566253662 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3.866955041885376 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3.866854190826416 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3.8667290210723877 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3.8670589923858643 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3.8667638301849365 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3.8669049739837646 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3.866889238357544 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3.8670589923858643 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3.866976022720337 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3.866912364959717 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3.8667821884155273 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3.866757869720459 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3.8670151233673096 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3.866995334625244 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3.8669216632843018 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3.8668010234832764 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3.8669803142547607 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3.8670432567596436 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3.8664190769195557 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3.866987705230713 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3.8669779300689697 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3.8670010566711426 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3.866955518722534 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3.866706609725952 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3.8669986724853516 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3.8669793605804443 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3.866959571838379 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3.866962194442749 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3.867098569869995 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3.866931200027466 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3.8665478229522705 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3.866999864578247 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3.8666982650756836 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3.8669779300689697 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3.8670384883880615 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3.866954803466797 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3.8669638633728027 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3.866706609725952 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3.866779327392578 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3.8665988445281982 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3.8669958114624023 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3.866950035095215 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3.866917848587036 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3.866973876953125 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3.867079019546509 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3.866995334625244 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3.867033004760742 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3.866969585418701 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3.867032051086426 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3.867035388946533 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3.8670854568481445 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3.8667314052581787 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3.8671748638153076 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3.8669769763946533 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3.867033004760742 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3.866990327835083 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3.8669662475585938 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3.8671324253082275 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3.866642951965332 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3.866832971572876 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3.8667237758636475 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3.8670384883880615 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3.8670082092285156 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3.8668737411499023 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3.8669822216033936 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3.8667140007019043 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3.866879940032959 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3.8669726848602295 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3.866973876953125 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3.8669960498809814 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3.8670871257781982 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3.8669381141662598 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3.8667569160461426 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3.866779327392578 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3.8666467666625977 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3.8669753074645996 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3.8670196533203125 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3.8669657707214355 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3.8670027256011963 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3.8666625022888184 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3.866672992706299 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3.866673707962036 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3.866952896118164 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3.866912364959717 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3.8669450283050537 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3.8669612407684326 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3.8666653633117676 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3.8666603565216064 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3.8667988777160645 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3.866905927658081 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3.8669886589050293 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3.8669209480285645 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3.8669302463531494 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3.8666305541992188 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3.866831064224243 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3.8667237758636475 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3.867032051086426 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3.866938591003418 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3.8669049739837646 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3.866960287094116 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3.8667407035827637 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3.8669593334198 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3.866588592529297 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3.8669769763946533 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3.866992473602295 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3.866990566253662 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3.867050886154175 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3.867063522338867 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3.8667333126068115 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3.867020606994629 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3.8670220375061035 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3.8669559955596924 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3.866995334625244 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3.86696720123291 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3.8666436672210693 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3.8666956424713135 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3.8668606281280518 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3.8669967651367188 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3.867050886154175 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3.8671352863311768 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3.8670666217803955 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3.8665802478790283 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3.8670554161071777 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3.86698842048645 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3.8670082092285156 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3.866926908493042 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3.8672401905059814 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3.8670403957366943 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3.866525173187256 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3.8667783737182617 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3.8672873973846436 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3.8669989109039307 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3.8670506477355957 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3.8670527935028076 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3.867067337036133 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3.866548776626587 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3.8670737743377686 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3.866884469985962 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3.8670146465301514 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3.8669841289520264 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3.867119789123535 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3.8669698238372803 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3.8663949966430664 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3.8669896125793457 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3.866818904876709 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3.866894483566284 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3.866882562637329 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3.8667519092559814 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3.8668718338012695 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3.8668456077575684 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3.866954803466797 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3.867034912109375 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3.866983652114868 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3.8669350147247314 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3.866926670074463 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3.8672127723693848 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3.8667821884155273 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3.8672850131988525 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3.866870880126953 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3.8669188022613525 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3.8668906688690186 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3.866818428039551 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3.8670101165771484 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3.8665950298309326 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3.8671324253082275 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3.8667352199554443 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3.866910457611084 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3.866947889328003 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3.8668580055236816 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3.867016553878784 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3.8664045333862305 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3.866908550262451 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3.8667988777160645 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3.867063045501709 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3.8669440746307373 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3.8668596744537354 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3.866786003112793 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3.8663010597229004 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3.86710524559021 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3.8669612407684326 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3.8669216632843018 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3.8669464588165283 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3.866957426071167 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3.8668980598449707 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3.8656322956085205 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3.866597890853882 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3.8668527603149414 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3.867367744445801 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3.8673739433288574 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3.866809368133545 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3.867016553878784 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3.8669490814208984 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3.8671114444732666 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3.867187023162842 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3.867084264755249 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3.8670272827148438 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3.866997003555298 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3.8670194149017334 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3.8663978576660156 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3.8668532371520996 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3.86665415763855 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3.866929292678833 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3.8669989109039307 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3.866837501525879 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3.8669850826263428 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3.865818500518799 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3.8666374683380127 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3.8667685985565186 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3.86702823638916 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3.8668882846832275 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3.866962194442749 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3.866821527481079 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3.8665425777435303 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3.8673489093780518 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3.866105318069458 - } - ] - }, - { - "bit": 6, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3.811953067779541 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3.812067747116089 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3.812023878097534 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 3.8120362758636475 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3.812218427658081 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3.812202215194702 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 3.8120412826538086 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3.812018632888794 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3.8119757175445557 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3.811859369277954 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3.8117969036102295 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3.811918258666992 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3.812103033065796 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3.8122310638427734 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3.811908721923828 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3.812023878097534 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3.812011241912842 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3.811892509460449 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3.8117971420288086 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3.8119747638702393 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3.8118393421173096 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3.812004566192627 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3.811976671218872 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 3.8120157718658447 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3.811953067779541 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3.8118245601654053 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3.8121347427368164 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3.8120522499084473 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3.8120148181915283 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3.8119101524353027 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3.8120365142822266 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3.8117425441741943 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3.8117568492889404 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3.8121540546417236 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3.8119711875915527 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3.812000036239624 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3.811941146850586 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3.8121237754821777 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3.8120956420898438 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3.8118433952331543 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3.812163829803467 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3.811863899230957 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3.8120720386505127 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3.812004566192627 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 3.811872959136963 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3.8120338916778564 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3.81187105178833 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3.8122856616973877 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3.812018632888794 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3.812044143676758 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3.812101125717163 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3.8120276927948 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3.8121256828308105 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3.811832904815674 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3.811948537826538 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3.812068223953247 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3.81197452545166 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3.8121230602264404 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3.8121182918548584 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3.811786651611328 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3.8117587566375732 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3.8115813732147217 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3.811863899230957 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3.8121345043182373 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3.8119964599609375 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3.812034845352173 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3.8120927810668945 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3.8117568492889404 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3.8120133876800537 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3.8119142055511475 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3.811976432800293 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3.8120574951171875 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3.8121933937072754 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3.8120884895324707 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3.8119540214538574 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3.8120179176330566 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3.812209367752075 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3.8120310306549072 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3.812004566192627 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3.812135696411133 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3.8119568824768066 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3.81164288520813 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3.8121092319488525 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3.8117592334747314 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3.812107563018799 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3.811912775039673 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3.8119378089904785 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3.8122520446777344 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3.811957359313965 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3.8120205402374268 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3.8119077682495117 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3.8119888305664062 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3.8120040893554688 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3.811908006668091 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3.8119266033172607 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3.811918258666992 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3.8117384910583496 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3.8119170665740967 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3.8119559288024902 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3.8120691776275635 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3.8119335174560547 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3.8120856285095215 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3.8118064403533936 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3.812035322189331 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3.811746597290039 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3.812074899673462 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3.8120667934417725 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3.811816692352295 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3.8121328353881836 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3.8119630813598633 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3.812140703201294 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3.812039852142334 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3.8119747638702393 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3.8119990825653076 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3.811976671218872 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3.8118319511413574 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3.8120064735412598 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3.8120226860046387 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3.8120529651641846 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3.8120484352111816 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3.8119702339172363 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3.8118464946746826 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3.8118247985839844 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3.811852216720581 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3.812107563018799 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3.8119611740112305 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3.812030076980591 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3.811927556991577 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3.8121631145477295 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3.812000036239624 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3.8119306564331055 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3.811715841293335 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3.8117470741271973 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3.812065839767456 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3.8120739459991455 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3.8121089935302734 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3.8120639324188232 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3.8119020462036133 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3.8118793964385986 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3.8118574619293213 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3.8118975162506104 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3.811980962753296 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3.8120970726013184 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3.812206506729126 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3.8120505809783936 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3.8118298053741455 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3.81208872795105 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3.812002182006836 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3.8120548725128174 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3.81180739402771 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3.8119940757751465 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3.812175989151001 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3.8119640350341797 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3.811868906021118 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3.8119118213653564 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3.811990261077881 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3.811938762664795 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3.812065839767456 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3.812082529067993 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3.811889410018921 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3.811990261077881 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3.8120357990264893 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3.81199312210083 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3.8120317459106445 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3.8119068145751953 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3.8120317459106445 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3.812030076980591 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3.812000274658203 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3.8119702339172363 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3.8120083808898926 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3.812089681625366 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3.8118093013763428 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3.81182861328125 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3.811796188354492 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3.8119757175445557 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3.811932325363159 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3.8120012283325195 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3.8117921352386475 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3.811983823776245 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3.8119313716888428 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3.8120555877685547 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3.812037229537964 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3.8120903968811035 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3.8121001720428467 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3.8120710849761963 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3.81196665763855 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3.8118913173675537 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3.8118176460266113 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3.8118481636047363 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3.8119068145751953 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3.8119754791259766 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3.811875104904175 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3.8119099140167236 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3.8119049072265625 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3.811943769454956 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3.812131881713867 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3.811983823776245 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3.811950922012329 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3.8120601177215576 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3.8120784759521484 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3.8121531009674072 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3.8119397163391113 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3.8120980262756348 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3.8120033740997314 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3.8119466304779053 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3.811765193939209 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3.8120179176330566 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3.8122012615203857 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3.812054395675659 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3.81227707862854 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3.8120975494384766 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3.8120310306549072 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3.8120012283325195 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3.812103509902954 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3.8119657039642334 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3.811952590942383 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3.8120064735412598 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3.8119871616363525 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3.811948537826538 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3.8120758533477783 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3.8120129108428955 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3.811995267868042 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3.8120758533477783 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3.8117847442626953 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3.8120229244232178 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3.812167167663574 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3.811932325363159 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3.8119776248931885 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3.811983823776245 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3.8118412494659424 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3.811840295791626 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3.8121092319488525 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3.812023639678955 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3.81183123588562 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3.812039375305176 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3.8116557598114014 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3.8119983673095703 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3.811959981918335 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3.8119776248931885 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3.811962366104126 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3.811899423599243 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3.8119406700134277 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3.8119540214538574 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3.8119847774505615 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3.8120083808898926 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3.812009334564209 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3.812039852142334 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3.811943769454956 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3.8120474815368652 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3.8118276596069336 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3.812000036239624 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3.8119940757751465 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3.8120062351226807 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3.8118784427642822 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3.811964988708496 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3.81189227104187 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3.811835765838623 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3.8121190071105957 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3.8117685317993164 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3.812039852142334 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3.8120548725128174 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3.8120148181915283 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3.811972141265869 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3.8119289875030518 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3.8120267391204834 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3.8118317127227783 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3.8120033740997314 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3.812138557434082 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3.8119492530822754 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3.8119912147521973 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3.811814069747925 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3.8119149208068848 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3.81201171875 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3.811953067779541 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3.8119397163391113 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3.8121907711029053 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3.8119847774505615 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3.811936616897583 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3.8120219707489014 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3.812148332595825 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3.8118813037872314 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3.811962127685547 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3.811882495880127 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3.8119871616363525 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3.8119313716888428 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3.8119425773620605 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3.811889410018921 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3.8118956089019775 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3.8119611740112305 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3.8118584156036377 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3.8121025562286377 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3.8119466304779053 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3.8119404315948486 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3.8118584156036377 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3.812068223953247 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3.8119418621063232 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3.812082052230835 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3.811946153640747 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3.8119993209838867 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3.8120903968811035 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3.8117775917053223 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3.811931610107422 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3.8118813037872314 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3.8119139671325684 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3.8119120597839355 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3.8118667602539062 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3.81195330619812 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3.8119561672210693 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3.8121285438537598 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3.8120357990264893 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3.8119571208953857 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3.8119053840637207 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3.8118855953216553 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3.811826705932617 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3.812061071395874 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3.8120176792144775 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3.8119847774505615 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3.8120710849761963 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3.8120837211608887 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3.8119447231292725 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3.8118317127227783 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3.811842203140259 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3.8120055198669434 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3.812004327774048 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3.812018632888794 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3.8119490146636963 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3.8117969036102295 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3.811997413635254 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3.8119304180145264 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3.8119819164276123 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3.8120739459991455 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3.811976909637451 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3.8118603229522705 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3.8119211196899414 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3.8119776248931885 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3.8116812705993652 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3.8119912147521973 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3.811985731124878 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3.812011241912842 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3.812183380126953 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3.8119568824768066 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3.8118700981140137 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3.811918020248413 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3.8120012283325195 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3.8119499683380127 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3.812065839767456 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3.812034845352173 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3.8119711875915527 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3.8119611740112305 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3.8119964599609375 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3.812000274658203 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3.812067747116089 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3.8120648860931396 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3.8119990825653076 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3.811889171600342 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3.8121747970581055 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3.811988592147827 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3.811986207962036 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3.8120148181915283 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3.81192946434021 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3.8120229244232178 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3.811805009841919 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3.8120052814483643 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3.8119826316833496 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3.8119993209838867 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3.812070846557617 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3.811978578567505 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3.812049388885498 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3.811901569366455 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3.811918258666992 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3.8117897510528564 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3.8120205402374268 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3.812091588973999 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3.8120803833007812 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3.8121068477630615 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3.811980962753296 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3.8119633197784424 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3.8120319843292236 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3.8119921684265137 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3.81209135055542 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3.811983585357666 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3.8120195865631104 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3.8119266033172607 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3.8121531009674072 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3.811922550201416 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3.8120574951171875 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3.8119728565216064 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3.812039375305176 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3.812119245529175 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3.8119256496429443 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3.81182861328125 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3.811680793762207 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3.812034845352173 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3.812011241912842 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3.8119161128997803 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3.811932325363159 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3.8118362426757812 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3.8118932247161865 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3.811666488647461 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3.8120129108428955 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3.812072992324829 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3.812044620513916 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3.8120760917663574 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3.811943769454956 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3.8121719360351562 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3.811946153640747 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3.8119940757751465 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3.8120012283325195 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3.8119921684265137 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3.8119871616363525 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3.8119871616363525 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3.8119583129882812 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3.8118112087249756 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3.8120005130767822 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3.81203293800354 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3.812000274658203 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3.8119540214538574 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3.8121912479400635 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3.812061071395874 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3.8117458820343018 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3.8120298385620117 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3.812049388885498 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3.812062978744507 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3.812103033065796 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3.8119003772735596 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3.8119139671325684 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3.8119444847106934 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3.8120412826538086 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3.8119983673095703 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3.8120594024658203 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3.8120205402374268 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3.8119711875915527 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3.8119795322418213 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3.811739444732666 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3.8120994567871094 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3.812002182006836 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3.8120267391204834 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3.8120384216308594 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3.8120055198669434 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3.8116984367370605 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3.8117785453796387 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3.812000274658203 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3.812039375305176 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3.8120741844177246 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3.812047243118286 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3.8117620944976807 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3.8120012283325195 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3.8117940425872803 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3.812026023864746 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3.8120760917663574 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3.8119256496429443 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3.8119728565216064 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3.81164288520813 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3.8120381832122803 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3.811826705932617 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3.8120267391204834 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3.8120229244232178 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3.8120474815368652 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3.812044143676758 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3.8120317459106445 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3.8118598461151123 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3.8121604919433594 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3.812023878097534 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3.8120362758636475 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3.811946153640747 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3.8119847774505615 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3.8117077350616455 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3.811798334121704 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3.8119797706604004 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3.8120431900024414 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3.812101125717163 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3.812138557434082 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3.8120710849761963 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3.8118412494659424 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3.812044620513916 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3.811950206756592 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3.8120052814483643 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3.812025785446167 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3.8119404315948486 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3.8119845390319824 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3.8118460178375244 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3.811918020248413 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3.8119702339172363 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3.812093734741211 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3.8121750354766846 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3.8119852542877197 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3.8120765686035156 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3.812150001525879 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3.811652183532715 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3.8120594024658203 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3.811958074569702 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3.811985731124878 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3.811957359313965 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3.8120975494384766 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3.811612129211426 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3.8118059635162354 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3.812114953994751 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3.8119559288024902 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3.8119239807128906 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3.811992645263672 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3.8118975162506104 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3.8119876384735107 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3.811938762664795 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3.8120083808898926 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3.811852216720581 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3.8120288848876953 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3.8121237754821777 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3.8120594024658203 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3.811544895172119 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3.8118984699249268 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3.8119912147521973 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3.8119819164276123 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3.811951160430908 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3.8121016025543213 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3.8120431900024414 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3.811936616897583 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3.811927556991577 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3.8120102882385254 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3.8118855953216553 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3.8120152950286865 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3.8118720054626465 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3.8119418621063232 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3.811483144760132 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3.81193470954895 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3.8120338916778564 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3.812011241912842 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3.811954975128174 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3.8117940425872803 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3.81189227104187 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3.811387777328491 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3.811798095703125 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3.8117294311523438 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3.8121025562286377 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3.8122661113739014 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3.812173843383789 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3.8120055198669434 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3.8114922046661377 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3.8120813369750977 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3.8118834495544434 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3.811973810195923 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3.812002420425415 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3.812131404876709 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3.812042236328125 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3.811657667160034 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3.8120083808898926 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3.811825752258301 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3.8120081424713135 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3.8119354248046875 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3.811699390411377 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3.8119394779205322 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3.81173038482666 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3.8116745948791504 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3.812105178833008 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3.8120031356811523 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3.8120102882385254 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3.8119466304779053 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3.812030792236328 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3.812103033065796 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3.812028408050537 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3.8117189407348633 - } - ] - }, - { - "bit": 7, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3.7788569927215576 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3.7788515090942383 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3.7789053916931152 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 3.7788496017456055 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3.778892755508423 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3.778860330581665 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 3.778761148452759 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3.778714418411255 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3.7788546085357666 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3.7787883281707764 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3.7785069942474365 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3.778850555419922 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3.7787721157073975 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3.778820037841797 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3.7787258625030518 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3.7787468433380127 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3.778681993484497 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3.7788171768188477 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3.7787508964538574 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3.77873158454895 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3.7789013385772705 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3.7787539958953857 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3.7787630558013916 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 3.7787132263183594 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3.778726577758789 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3.778618574142456 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3.7784905433654785 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3.77874493598938 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3.7786648273468018 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3.7786591053009033 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3.778801918029785 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3.77878475189209 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3.77899432182312 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3.778766632080078 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3.7788162231445312 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3.778797149658203 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3.7787296772003174 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3.7786829471588135 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3.7788567543029785 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3.7786824703216553 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3.7785558700561523 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3.7789838314056396 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3.7789037227630615 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3.7788302898406982 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 3.7789392471313477 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3.7790157794952393 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3.7785587310791016 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3.778400421142578 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3.7786033153533936 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3.778892755508423 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3.778839588165283 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3.7785966396331787 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3.7789058685302734 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3.778653621673584 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3.7786612510681152 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3.7787530422210693 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3.778815269470215 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3.7788143157958984 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3.778920888900757 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3.7787883281707764 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3.7786970138549805 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3.778644561767578 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3.7787630558013916 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3.778822183609009 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3.7787559032440186 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3.7788407802581787 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3.7790017127990723 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3.778625011444092 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3.778693914413452 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3.7786378860473633 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3.778810977935791 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3.77893328666687 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3.778749465942383 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3.778764486312866 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3.7787044048309326 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3.7785332202911377 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3.778604507446289 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3.7788009643554688 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3.7788004875183105 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3.7787091732025146 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3.778820514678955 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3.7788045406341553 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3.7790114879608154 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3.7785675525665283 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3.778764009475708 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3.7787692546844482 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3.778848648071289 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3.7788569927215576 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3.7786316871643066 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3.778681993484497 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3.778670310974121 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3.778895854949951 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3.778808116912842 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3.778862953186035 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3.778839588165283 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3.7788829803466797 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3.779042959213257 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3.778642177581787 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3.778795003890991 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3.7788515090942383 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3.7789573669433594 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3.7787153720855713 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3.778670310974121 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3.7787399291992188 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3.7788212299346924 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3.7789251804351807 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3.7788610458374023 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3.7787561416625977 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3.778777837753296 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3.778862953186035 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3.779005527496338 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3.7786693572998047 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3.778872013092041 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3.778799533843994 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3.7788174152374268 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3.778820037841797 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3.7788522243499756 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3.7788684368133545 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3.77880597114563 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3.778775215148926 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3.7787530422210693 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3.7788515090942383 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3.778984785079956 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3.778782844543457 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3.7789249420166016 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3.778827428817749 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3.7787630558013916 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3.7785487174987793 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3.778865098953247 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3.778604507446289 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3.7786495685577393 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3.7788891792297363 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3.7789080142974854 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3.7789089679718018 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3.778848886489868 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3.7787277698516846 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3.7787609100341797 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3.778838634490967 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3.778797149658203 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3.778801918029785 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3.7787630558013916 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3.7786495685577393 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3.778679132461548 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3.7787837982177734 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3.7787656784057617 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3.7788162231445312 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3.7788679599761963 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3.7787415981292725 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3.7787909507751465 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3.7785935401916504 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3.7787487506866455 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3.7787399291992188 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3.778813362121582 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3.778773069381714 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3.7789435386657715 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3.778796434402466 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3.7786169052124023 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3.7788140773773193 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3.7787134647369385 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3.7787649631500244 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3.7788124084472656 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3.778839588165283 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3.778897285461426 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3.778655767440796 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3.778895378112793 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3.7787296772003174 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3.7787890434265137 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3.7787258625030518 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3.7789053916931152 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3.7788140773773193 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3.7788143157958984 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3.778994560241699 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3.7787346839904785 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3.778754949569702 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3.7788305282592773 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3.7787938117980957 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3.778728723526001 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3.7788426876068115 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3.778846502304077 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3.7788333892822266 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3.7788751125335693 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3.7787439823150635 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3.7789392471313477 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3.7788233757019043 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3.7786521911621094 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3.77888560295105 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3.778801202774048 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3.77874493598938 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3.778728723526001 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3.7788305282592773 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3.7788028717041016 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3.778754949569702 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3.7787318229675293 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3.778663158416748 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3.778749942779541 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3.7787365913391113 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3.7787187099456787 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3.7787766456604004 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3.778703451156616 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3.7789385318756104 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3.778743028640747 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3.778750419616699 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3.778801202774048 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3.7788383960723877 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3.7788119316101074 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3.7785496711730957 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3.7787539958953857 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3.778663158416748 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3.778803586959839 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3.778757095336914 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3.7787938117980957 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3.778728723526001 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3.778687000274658 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3.778872489929199 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3.7787559032440186 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3.77878999710083 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3.778822422027588 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3.778944253921509 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3.7789130210876465 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3.778846502304077 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3.778843402862549 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3.7788619995117188 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3.778749465942383 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3.7788774967193604 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3.7789242267608643 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3.778902769088745 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3.7786293029785156 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3.7788262367248535 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3.778881072998047 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3.7788071632385254 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3.7789111137390137 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3.778921127319336 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3.778857707977295 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3.778954029083252 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3.778743028640747 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3.7788424491882324 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3.77868914604187 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3.778803586959839 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3.7786757946014404 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3.778670310974121 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3.77895450592041 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3.7789466381073 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3.7787978649139404 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3.7788312435150146 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3.7787468433380127 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3.7788164615631104 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3.7787938117980957 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3.778658628463745 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3.778761625289917 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3.7787559032440186 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3.7788326740264893 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3.778864860534668 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3.7787816524505615 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3.778895854949951 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3.778623580932617 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3.778773784637451 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3.7786340713500977 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3.77878475189209 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3.7788233757019043 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3.7788071632385254 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3.7789371013641357 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3.778762102127075 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3.778839588165283 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3.778897285461426 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3.778902769088745 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3.778799057006836 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3.7788004875183105 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3.778813362121582 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3.7787585258483887 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3.7787938117980957 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3.778740882873535 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3.778691053390503 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3.7787866592407227 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3.7785513401031494 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3.7787692546844482 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3.7787253856658936 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3.778838634490967 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3.778728723526001 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3.7788872718811035 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3.7788333892822266 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3.7785911560058594 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3.7787747383117676 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3.7787020206451416 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3.7787044048309326 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3.7787375450134277 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3.778801202774048 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3.778872013092041 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3.7787585258483887 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3.778797149658203 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3.778866767883301 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3.7786717414855957 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3.7788021564483643 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3.7788171768188477 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3.778749465942383 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3.778866767883301 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3.7785446643829346 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3.7786717414855957 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3.77880597114563 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3.7787630558013916 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3.77878475189209 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3.778782844543457 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3.7787487506866455 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3.77868914604187 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3.7787697315216064 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3.77873158454895 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3.7789666652679443 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3.7788610458374023 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3.7789313793182373 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3.7788476943969727 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3.7787256240844727 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3.778839588165283 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3.7785842418670654 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3.778754949569702 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3.778740882873535 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3.7788476943969727 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3.77884578704834 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3.778852939605713 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3.778815984725952 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3.778855800628662 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3.7787978649139404 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3.778799057006836 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3.7788619995117188 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3.7788357734680176 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3.7786879539489746 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3.77874493598938 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3.7786905765533447 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3.7788150310516357 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3.7787349224090576 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3.7788333892822266 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3.7788305282592773 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3.7787253856658936 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3.778698205947876 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3.7788143157958984 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3.7787981033325195 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3.778737783432007 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3.7788124084472656 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3.7787513732910156 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3.7787153720855713 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3.778865098953247 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3.778749465942383 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3.778756856918335 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3.7787652015686035 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3.7787771224975586 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3.7787530422210693 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3.778862953186035 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3.7786941528320312 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3.77864408493042 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3.778764486312866 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3.7787530422210693 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3.7787258625030518 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3.7787671089172363 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3.7788264751434326 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3.778696298599243 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3.7786431312561035 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3.7787020206451416 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3.7788243293762207 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3.7788522243499756 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3.77866268157959 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3.7788496017456055 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3.778414487838745 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3.778841495513916 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3.7788245677948 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3.7788100242614746 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3.7788870334625244 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3.778730630874634 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3.7787938117980957 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3.778993606567383 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3.778714418411255 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3.7787716388702393 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3.7787539958953857 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3.7787134647369385 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3.7787253856658936 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3.7788071632385254 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3.7786593437194824 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3.778813362121582 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3.7787721157073975 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3.7787609100341797 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3.7787747383117676 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3.7787399291992188 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3.7786591053009033 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3.778780221939087 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3.7788643836975098 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3.778796434402466 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3.7788569927215576 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3.778759479522705 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3.778958559036255 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3.7787206172943115 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3.7785544395446777 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3.778843641281128 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3.7788352966308594 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3.7788243293762207 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3.7786552906036377 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3.778730630874634 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3.778820276260376 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3.7788000106811523 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3.7788150310516357 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3.77882719039917 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3.778879165649414 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3.7787227630615234 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3.7786004543304443 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3.77878737449646 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3.7787468433380127 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3.7787792682647705 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3.7788093090057373 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3.778813362121582 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3.778630018234253 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3.7787258625030518 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3.77878475189209 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3.778768539428711 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3.778787851333618 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3.7787468433380127 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3.7788476943969727 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3.7786002159118652 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3.7786953449249268 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3.778782844543457 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3.778797149658203 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3.778838634490967 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3.778749942779541 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3.7786738872528076 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3.778701066970825 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3.778590202331543 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3.778759479522705 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3.7787907123565674 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3.7787928581237793 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3.778747797012329 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3.7788314819335938 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3.778787851333618 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3.778700113296509 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3.778947591781616 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3.7787561416625977 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3.778771162033081 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3.778780937194824 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3.7788286209106445 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3.7786848545074463 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3.7787296772003174 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3.7788000106811523 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3.778862953186035 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3.778757095336914 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3.7788052558898926 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3.778820514678955 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3.778625726699829 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3.77864408493042 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3.778822422027588 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3.778761625289917 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3.7787423133850098 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3.7787981033325195 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3.778754949569702 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3.7785487174987793 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3.7787225246429443 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3.778674840927124 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3.778809070587158 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3.7788100242614746 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3.778754949569702 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3.778762102127075 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3.7785167694091797 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3.778796434402466 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3.7784907817840576 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3.7787325382232666 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3.7788281440734863 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3.7787909507751465 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3.7787585258483887 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3.7788658142089844 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3.7787954807281494 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3.7787041664123535 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3.778773069381714 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3.7787747383117676 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3.778820514678955 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3.778797149658203 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3.778749942779541 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3.778629779815674 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3.778740644454956 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3.778721809387207 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3.7787585258483887 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3.7788050174713135 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3.778773784637451 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3.778735637664795 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3.7785487174987793 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3.7785468101501465 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3.7788586616516113 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3.778866767883301 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3.7788617610931396 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3.7787909507751465 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3.7785801887512207 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3.778791904449463 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3.778590202331543 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3.7787044048309326 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3.7788143157958984 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3.7787296772003174 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3.7787041664123535 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3.7784793376922607 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3.7786097526550293 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3.778527021408081 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3.7787468433380127 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3.778764009475708 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3.7787554264068604 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3.778848886489868 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3.7788143157958984 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3.7788305282592773 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3.7787487506866455 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3.778724431991577 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3.7786784172058105 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3.778712511062622 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3.7787325382232666 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3.7786707878112793 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3.7784423828125 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3.7786951065063477 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3.7787458896636963 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3.7787091732025146 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3.7788305282592773 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3.7788233757019043 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3.7786824703216553 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3.778799533843994 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3.7786810398101807 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3.778766632080078 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3.7786829471588135 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3.77878999710083 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3.7787423133850098 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3.778615951538086 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3.7787721157073975 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3.7787091732025146 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3.7787671089172363 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3.778799533843994 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3.7788405418395996 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3.7788679599761963 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3.7784459590911865 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3.7787163257598877 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3.778963565826416 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3.778806209564209 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3.7787439823150635 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3.7787346839904785 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3.778825283050537 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3.778806209564209 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3.778815984725952 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3.778740882873535 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3.7788326740264893 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3.7787437438964844 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3.7788171768188477 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3.778773307800293 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3.778698205947876 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3.7786779403686523 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3.7786593437194824 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3.7787206172943115 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3.7787134647369385 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3.778733730316162 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3.77873158454895 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3.778473138809204 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3.778780221939087 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3.7786571979522705 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3.7786991596221924 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3.7787489891052246 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3.778804063796997 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3.778677463531494 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3.7788352966308594 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3.7787277698516846 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3.7790026664733887 - } - ] - }, - { - "bit": 8, - "ppls": [ - { - "layer": "layer.0.self_attn.q_proj", - "ppl": 3.7587106227874756 - }, - { - "layer": "layer.0.self_attn.k_proj", - "ppl": 3.7588627338409424 - }, - { - "layer": "layer.0.self_attn.v_proj", - "ppl": 3.758840560913086 - }, - { - "layer": "layer.0.self_attn.o_proj", - "ppl": 3.758925199508667 - }, - { - "layer": "layer.0.mlp.gate_proj", - "ppl": 3.75884747505188 - }, - { - "layer": "layer.0.mlp.up_proj", - "ppl": 3.758901834487915 - }, - { - "layer": "layer.0.mlp.down_proj", - "ppl": 3.7587947845458984 - }, - { - "layer": "layer.1.self_attn.q_proj", - "ppl": 3.758840560913086 - }, - { - "layer": "layer.1.self_attn.k_proj", - "ppl": 3.7587900161743164 - }, - { - "layer": "layer.1.self_attn.v_proj", - "ppl": 3.7588655948638916 - }, - { - "layer": "layer.1.self_attn.o_proj", - "ppl": 3.758852005004883 - }, - { - "layer": "layer.1.mlp.gate_proj", - "ppl": 3.7588412761688232 - }, - { - "layer": "layer.1.mlp.up_proj", - "ppl": 3.7588279247283936 - }, - { - "layer": "layer.1.mlp.down_proj", - "ppl": 3.7588601112365723 - }, - { - "layer": "layer.2.self_attn.q_proj", - "ppl": 3.758765935897827 - }, - { - "layer": "layer.2.self_attn.k_proj", - "ppl": 3.7588984966278076 - }, - { - "layer": "layer.2.self_attn.v_proj", - "ppl": 3.7588279247283936 - }, - { - "layer": "layer.2.self_attn.o_proj", - "ppl": 3.7587950229644775 - }, - { - "layer": "layer.2.mlp.gate_proj", - "ppl": 3.7587051391601562 - }, - { - "layer": "layer.2.mlp.up_proj", - "ppl": 3.7589309215545654 - }, - { - "layer": "layer.2.mlp.down_proj", - "ppl": 3.758718967437744 - }, - { - "layer": "layer.3.self_attn.q_proj", - "ppl": 3.7587435245513916 - }, - { - "layer": "layer.3.self_attn.k_proj", - "ppl": 3.75888729095459 - }, - { - "layer": "layer.3.self_attn.v_proj", - "ppl": 3.7587695121765137 - }, - { - "layer": "layer.3.self_attn.o_proj", - "ppl": 3.7589104175567627 - }, - { - "layer": "layer.3.mlp.gate_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.3.mlp.up_proj", - "ppl": 3.7588951587677 - }, - { - "layer": "layer.3.mlp.down_proj", - "ppl": 3.7586288452148438 - }, - { - "layer": "layer.4.self_attn.q_proj", - "ppl": 3.7587363719940186 - }, - { - "layer": "layer.4.self_attn.k_proj", - "ppl": 3.758758783340454 - }, - { - "layer": "layer.4.self_attn.v_proj", - "ppl": 3.7588679790496826 - }, - { - "layer": "layer.4.self_attn.o_proj", - "ppl": 3.758828639984131 - }, - { - "layer": "layer.4.mlp.gate_proj", - "ppl": 3.7587642669677734 - }, - { - "layer": "layer.4.mlp.up_proj", - "ppl": 3.758671283721924 - }, - { - "layer": "layer.4.mlp.down_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.5.self_attn.q_proj", - "ppl": 3.7587578296661377 - }, - { - "layer": "layer.5.self_attn.k_proj", - "ppl": 3.7587857246398926 - }, - { - "layer": "layer.5.self_attn.v_proj", - "ppl": 3.7587740421295166 - }, - { - "layer": "layer.5.self_attn.o_proj", - "ppl": 3.758713960647583 - }, - { - "layer": "layer.5.mlp.gate_proj", - "ppl": 3.7588202953338623 - }, - { - "layer": "layer.5.mlp.up_proj", - "ppl": 3.758861780166626 - }, - { - "layer": "layer.5.mlp.down_proj", - "ppl": 3.758666515350342 - }, - { - "layer": "layer.6.self_attn.q_proj", - "ppl": 3.7588205337524414 - }, - { - "layer": "layer.6.self_attn.k_proj", - "ppl": 3.758760690689087 - }, - { - "layer": "layer.6.self_attn.v_proj", - "ppl": 3.7587971687316895 - }, - { - "layer": "layer.6.self_attn.o_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.6.mlp.gate_proj", - "ppl": 3.7586934566497803 - }, - { - "layer": "layer.6.mlp.up_proj", - "ppl": 3.7588746547698975 - }, - { - "layer": "layer.6.mlp.down_proj", - "ppl": 3.7588584423065186 - }, - { - "layer": "layer.7.self_attn.q_proj", - "ppl": 3.758850336074829 - }, - { - "layer": "layer.7.self_attn.k_proj", - "ppl": 3.7587625980377197 - }, - { - "layer": "layer.7.self_attn.v_proj", - "ppl": 3.7589075565338135 - }, - { - "layer": "layer.7.self_attn.o_proj", - "ppl": 3.7588160037994385 - }, - { - "layer": "layer.7.mlp.gate_proj", - "ppl": 3.7588043212890625 - }, - { - "layer": "layer.7.mlp.up_proj", - "ppl": 3.758845806121826 - }, - { - "layer": "layer.7.mlp.down_proj", - "ppl": 3.7588932514190674 - }, - { - "layer": "layer.8.self_attn.q_proj", - "ppl": 3.7588350772857666 - }, - { - "layer": "layer.8.self_attn.k_proj", - "ppl": 3.7588672637939453 - }, - { - "layer": "layer.8.self_attn.v_proj", - "ppl": 3.758913516998291 - }, - { - "layer": "layer.8.self_attn.o_proj", - "ppl": 3.7588162422180176 - }, - { - "layer": "layer.8.mlp.gate_proj", - "ppl": 3.75869083404541 - }, - { - "layer": "layer.8.mlp.up_proj", - "ppl": 3.758758544921875 - }, - { - "layer": "layer.8.mlp.down_proj", - "ppl": 3.758735179901123 - }, - { - "layer": "layer.9.self_attn.q_proj", - "ppl": 3.7587404251098633 - }, - { - "layer": "layer.9.self_attn.k_proj", - "ppl": 3.7588484287261963 - }, - { - "layer": "layer.9.self_attn.v_proj", - "ppl": 3.758755683898926 - }, - { - "layer": "layer.9.self_attn.o_proj", - "ppl": 3.758591651916504 - }, - { - "layer": "layer.9.mlp.gate_proj", - "ppl": 3.7586774826049805 - }, - { - "layer": "layer.9.mlp.up_proj", - "ppl": 3.758845806121826 - }, - { - "layer": "layer.9.mlp.down_proj", - "ppl": 3.758667469024658 - }, - { - "layer": "layer.10.self_attn.q_proj", - "ppl": 3.758753538131714 - }, - { - "layer": "layer.10.self_attn.k_proj", - "ppl": 3.758822441101074 - }, - { - "layer": "layer.10.self_attn.v_proj", - "ppl": 3.7587597370147705 - }, - { - "layer": "layer.10.self_attn.o_proj", - "ppl": 3.758904457092285 - }, - { - "layer": "layer.10.mlp.gate_proj", - "ppl": 3.7586240768432617 - }, - { - "layer": "layer.10.mlp.up_proj", - "ppl": 3.758812427520752 - }, - { - "layer": "layer.10.mlp.down_proj", - "ppl": 3.758981227874756 - }, - { - "layer": "layer.11.self_attn.q_proj", - "ppl": 3.758784770965576 - }, - { - "layer": "layer.11.self_attn.k_proj", - "ppl": 3.758882761001587 - }, - { - "layer": "layer.11.self_attn.v_proj", - "ppl": 3.7588119506835938 - }, - { - "layer": "layer.11.self_attn.o_proj", - "ppl": 3.7587239742279053 - }, - { - "layer": "layer.11.mlp.gate_proj", - "ppl": 3.7586398124694824 - }, - { - "layer": "layer.11.mlp.up_proj", - "ppl": 3.758680582046509 - }, - { - "layer": "layer.11.mlp.down_proj", - "ppl": 3.758802890777588 - }, - { - "layer": "layer.12.self_attn.q_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.12.self_attn.k_proj", - "ppl": 3.758790969848633 - }, - { - "layer": "layer.12.self_attn.v_proj", - "ppl": 3.7588515281677246 - }, - { - "layer": "layer.12.self_attn.o_proj", - "ppl": 3.7588233947753906 - }, - { - "layer": "layer.12.mlp.gate_proj", - "ppl": 3.7586686611175537 - }, - { - "layer": "layer.12.mlp.up_proj", - "ppl": 3.758671760559082 - }, - { - "layer": "layer.12.mlp.down_proj", - "ppl": 3.7588090896606445 - }, - { - "layer": "layer.13.self_attn.q_proj", - "ppl": 3.758824586868286 - }, - { - "layer": "layer.13.self_attn.k_proj", - "ppl": 3.7588624954223633 - }, - { - "layer": "layer.13.self_attn.v_proj", - "ppl": 3.7587037086486816 - }, - { - "layer": "layer.13.self_attn.o_proj", - "ppl": 3.7587947845458984 - }, - { - "layer": "layer.13.mlp.gate_proj", - "ppl": 3.7586286067962646 - }, - { - "layer": "layer.13.mlp.up_proj", - "ppl": 3.7585296630859375 - }, - { - "layer": "layer.13.mlp.down_proj", - "ppl": 3.7588608264923096 - }, - { - "layer": "layer.14.self_attn.q_proj", - "ppl": 3.7587695121765137 - }, - { - "layer": "layer.14.self_attn.k_proj", - "ppl": 3.758850336074829 - }, - { - "layer": "layer.14.self_attn.v_proj", - "ppl": 3.7587668895721436 - }, - { - "layer": "layer.14.self_attn.o_proj", - "ppl": 3.7588751316070557 - }, - { - "layer": "layer.14.mlp.gate_proj", - "ppl": 3.758756160736084 - }, - { - "layer": "layer.14.mlp.up_proj", - "ppl": 3.7587947845458984 - }, - { - "layer": "layer.14.mlp.down_proj", - "ppl": 3.7586727142333984 - }, - { - "layer": "layer.15.self_attn.q_proj", - "ppl": 3.7588181495666504 - }, - { - "layer": "layer.15.self_attn.k_proj", - "ppl": 3.758835792541504 - }, - { - "layer": "layer.15.self_attn.v_proj", - "ppl": 3.7587032318115234 - }, - { - "layer": "layer.15.self_attn.o_proj", - "ppl": 3.7588279247283936 - }, - { - "layer": "layer.15.mlp.gate_proj", - "ppl": 3.7587625980377197 - }, - { - "layer": "layer.15.mlp.up_proj", - "ppl": 3.758730173110962 - }, - { - "layer": "layer.15.mlp.down_proj", - "ppl": 3.758859395980835 - }, - { - "layer": "layer.16.self_attn.q_proj", - "ppl": 3.7587838172912598 - }, - { - "layer": "layer.16.self_attn.k_proj", - "ppl": 3.7587618827819824 - }, - { - "layer": "layer.16.self_attn.v_proj", - "ppl": 3.7587859630584717 - }, - { - "layer": "layer.16.self_attn.o_proj", - "ppl": 3.7587363719940186 - }, - { - "layer": "layer.16.mlp.gate_proj", - "ppl": 3.758742094039917 - }, - { - "layer": "layer.16.mlp.up_proj", - "ppl": 3.7586960792541504 - }, - { - "layer": "layer.16.mlp.down_proj", - "ppl": 3.758917808532715 - }, - { - "layer": "layer.17.self_attn.q_proj", - "ppl": 3.758779525756836 - }, - { - "layer": "layer.17.self_attn.k_proj", - "ppl": 3.758758783340454 - }, - { - "layer": "layer.17.self_attn.v_proj", - "ppl": 3.758742570877075 - }, - { - "layer": "layer.17.self_attn.o_proj", - "ppl": 3.758726119995117 - }, - { - "layer": "layer.17.mlp.gate_proj", - "ppl": 3.7587575912475586 - }, - { - "layer": "layer.17.mlp.up_proj", - "ppl": 3.7586781978607178 - }, - { - "layer": "layer.17.mlp.down_proj", - "ppl": 3.7588555812835693 - }, - { - "layer": "layer.18.self_attn.q_proj", - "ppl": 3.758812427520752 - }, - { - "layer": "layer.18.self_attn.k_proj", - "ppl": 3.7588601112365723 - }, - { - "layer": "layer.18.self_attn.v_proj", - "ppl": 3.7588484287261963 - }, - { - "layer": "layer.18.self_attn.o_proj", - "ppl": 3.7587900161743164 - }, - { - "layer": "layer.18.mlp.gate_proj", - "ppl": 3.7586662769317627 - }, - { - "layer": "layer.18.mlp.up_proj", - "ppl": 3.758897304534912 - }, - { - "layer": "layer.18.mlp.down_proj", - "ppl": 3.758810043334961 - }, - { - "layer": "layer.19.self_attn.q_proj", - "ppl": 3.75874662399292 - }, - { - "layer": "layer.19.self_attn.k_proj", - "ppl": 3.758774518966675 - }, - { - "layer": "layer.19.self_attn.v_proj", - "ppl": 3.758758783340454 - }, - { - "layer": "layer.19.self_attn.o_proj", - "ppl": 3.7588000297546387 - }, - { - "layer": "layer.19.mlp.gate_proj", - "ppl": 3.7587482929229736 - }, - { - "layer": "layer.19.mlp.up_proj", - "ppl": 3.7587006092071533 - }, - { - "layer": "layer.19.mlp.down_proj", - "ppl": 3.7585248947143555 - }, - { - "layer": "layer.20.self_attn.q_proj", - "ppl": 3.758824586868286 - }, - { - "layer": "layer.20.self_attn.k_proj", - "ppl": 3.7587382793426514 - }, - { - "layer": "layer.20.self_attn.v_proj", - "ppl": 3.758828639984131 - }, - { - "layer": "layer.20.self_attn.o_proj", - "ppl": 3.758826971054077 - }, - { - "layer": "layer.20.mlp.gate_proj", - "ppl": 3.7587146759033203 - }, - { - "layer": "layer.20.mlp.up_proj", - "ppl": 3.7589049339294434 - }, - { - "layer": "layer.20.mlp.down_proj", - "ppl": 3.758777618408203 - }, - { - "layer": "layer.21.self_attn.q_proj", - "ppl": 3.758758783340454 - }, - { - "layer": "layer.21.self_attn.k_proj", - "ppl": 3.7587990760803223 - }, - { - "layer": "layer.21.self_attn.v_proj", - "ppl": 3.7587926387786865 - }, - { - "layer": "layer.21.self_attn.o_proj", - "ppl": 3.7587625980377197 - }, - { - "layer": "layer.21.mlp.gate_proj", - "ppl": 3.758812427520752 - }, - { - "layer": "layer.21.mlp.up_proj", - "ppl": 3.758850574493408 - }, - { - "layer": "layer.21.mlp.down_proj", - "ppl": 3.7587695121765137 - }, - { - "layer": "layer.22.self_attn.q_proj", - "ppl": 3.758786678314209 - }, - { - "layer": "layer.22.self_attn.k_proj", - "ppl": 3.7587578296661377 - }, - { - "layer": "layer.22.self_attn.v_proj", - "ppl": 3.758819341659546 - }, - { - "layer": "layer.22.self_attn.o_proj", - "ppl": 3.7587592601776123 - }, - { - "layer": "layer.22.mlp.gate_proj", - "ppl": 3.7588980197906494 - }, - { - "layer": "layer.22.mlp.up_proj", - "ppl": 3.758859395980835 - }, - { - "layer": "layer.22.mlp.down_proj", - "ppl": 3.7589752674102783 - }, - { - "layer": "layer.23.self_attn.q_proj", - "ppl": 3.758784532546997 - }, - { - "layer": "layer.23.self_attn.k_proj", - "ppl": 3.7586934566497803 - }, - { - "layer": "layer.23.self_attn.v_proj", - "ppl": 3.758728265762329 - }, - { - "layer": "layer.23.self_attn.o_proj", - "ppl": 3.75876522064209 - }, - { - "layer": "layer.23.mlp.gate_proj", - "ppl": 3.758845806121826 - }, - { - "layer": "layer.23.mlp.up_proj", - "ppl": 3.7588438987731934 - }, - { - "layer": "layer.23.mlp.down_proj", - "ppl": 3.7587482929229736 - }, - { - "layer": "layer.24.self_attn.q_proj", - "ppl": 3.7587263584136963 - }, - { - "layer": "layer.24.self_attn.k_proj", - "ppl": 3.7587835788726807 - }, - { - "layer": "layer.24.self_attn.v_proj", - "ppl": 3.7587058544158936 - }, - { - "layer": "layer.24.self_attn.o_proj", - "ppl": 3.758793830871582 - }, - { - "layer": "layer.24.mlp.gate_proj", - "ppl": 3.7587249279022217 - }, - { - "layer": "layer.24.mlp.up_proj", - "ppl": 3.7586405277252197 - }, - { - "layer": "layer.24.mlp.down_proj", - "ppl": 3.758779764175415 - }, - { - "layer": "layer.25.self_attn.q_proj", - "ppl": 3.758821487426758 - }, - { - "layer": "layer.25.self_attn.k_proj", - "ppl": 3.758723020553589 - }, - { - "layer": "layer.25.self_attn.v_proj", - "ppl": 3.758694887161255 - }, - { - "layer": "layer.25.self_attn.o_proj", - "ppl": 3.7587404251098633 - }, - { - "layer": "layer.25.mlp.gate_proj", - "ppl": 3.758582353591919 - }, - { - "layer": "layer.25.mlp.up_proj", - "ppl": 3.7588112354278564 - }, - { - "layer": "layer.25.mlp.down_proj", - "ppl": 3.758795976638794 - }, - { - "layer": "layer.26.self_attn.q_proj", - "ppl": 3.7587344646453857 - }, - { - "layer": "layer.26.self_attn.k_proj", - "ppl": 3.758777618408203 - }, - { - "layer": "layer.26.self_attn.v_proj", - "ppl": 3.758789300918579 - }, - { - "layer": "layer.26.self_attn.o_proj", - "ppl": 3.758723020553589 - }, - { - "layer": "layer.26.mlp.gate_proj", - "ppl": 3.7587006092071533 - }, - { - "layer": "layer.26.mlp.up_proj", - "ppl": 3.7587077617645264 - }, - { - "layer": "layer.26.mlp.down_proj", - "ppl": 3.758680582046509 - }, - { - "layer": "layer.27.self_attn.q_proj", - "ppl": 3.75876522064209 - }, - { - "layer": "layer.27.self_attn.k_proj", - "ppl": 3.7588555812835693 - }, - { - "layer": "layer.27.self_attn.v_proj", - "ppl": 3.7587344646453857 - }, - { - "layer": "layer.27.self_attn.o_proj", - "ppl": 3.758831024169922 - }, - { - "layer": "layer.27.mlp.gate_proj", - "ppl": 3.758746385574341 - }, - { - "layer": "layer.27.mlp.up_proj", - "ppl": 3.758681535720825 - }, - { - "layer": "layer.27.mlp.down_proj", - "ppl": 3.7587435245513916 - }, - { - "layer": "layer.28.self_attn.q_proj", - "ppl": 3.7587735652923584 - }, - { - "layer": "layer.28.self_attn.k_proj", - "ppl": 3.758852005004883 - }, - { - "layer": "layer.28.self_attn.v_proj", - "ppl": 3.7587578296661377 - }, - { - "layer": "layer.28.self_attn.o_proj", - "ppl": 3.7588083744049072 - }, - { - "layer": "layer.28.mlp.gate_proj", - "ppl": 3.758821487426758 - }, - { - "layer": "layer.28.mlp.up_proj", - "ppl": 3.758685350418091 - }, - { - "layer": "layer.28.mlp.down_proj", - "ppl": 3.7588438987731934 - }, - { - "layer": "layer.29.self_attn.q_proj", - "ppl": 3.7587828636169434 - }, - { - "layer": "layer.29.self_attn.k_proj", - "ppl": 3.7588019371032715 - }, - { - "layer": "layer.29.self_attn.v_proj", - "ppl": 3.75879168510437 - }, - { - "layer": "layer.29.self_attn.o_proj", - "ppl": 3.7588000297546387 - }, - { - "layer": "layer.29.mlp.gate_proj", - "ppl": 3.758948802947998 - }, - { - "layer": "layer.29.mlp.up_proj", - "ppl": 3.758718967437744 - }, - { - "layer": "layer.29.mlp.down_proj", - "ppl": 3.758888006210327 - }, - { - "layer": "layer.30.self_attn.q_proj", - "ppl": 3.758828639984131 - }, - { - "layer": "layer.30.self_attn.k_proj", - "ppl": 3.7588062286376953 - }, - { - "layer": "layer.30.self_attn.v_proj", - "ppl": 3.758786916732788 - }, - { - "layer": "layer.30.self_attn.o_proj", - "ppl": 3.7587482929229736 - }, - { - "layer": "layer.30.mlp.gate_proj", - "ppl": 3.7586684226989746 - }, - { - "layer": "layer.30.mlp.up_proj", - "ppl": 3.758707284927368 - }, - { - "layer": "layer.30.mlp.down_proj", - "ppl": 3.7587273120880127 - }, - { - "layer": "layer.31.self_attn.q_proj", - "ppl": 3.758739471435547 - }, - { - "layer": "layer.31.self_attn.k_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.31.self_attn.v_proj", - "ppl": 3.7586028575897217 - }, - { - "layer": "layer.31.self_attn.o_proj", - "ppl": 3.7587928771972656 - }, - { - "layer": "layer.31.mlp.gate_proj", - "ppl": 3.7586605548858643 - }, - { - "layer": "layer.31.mlp.up_proj", - "ppl": 3.758843183517456 - }, - { - "layer": "layer.31.mlp.down_proj", - "ppl": 3.758587598800659 - }, - { - "layer": "layer.32.self_attn.q_proj", - "ppl": 3.758885145187378 - }, - { - "layer": "layer.32.self_attn.k_proj", - "ppl": 3.7589690685272217 - }, - { - "layer": "layer.32.self_attn.v_proj", - "ppl": 3.758805274963379 - }, - { - "layer": "layer.32.self_attn.o_proj", - "ppl": 3.758723258972168 - }, - { - "layer": "layer.32.mlp.gate_proj", - "ppl": 3.758802890777588 - }, - { - "layer": "layer.32.mlp.up_proj", - "ppl": 3.758687973022461 - }, - { - "layer": "layer.32.mlp.down_proj", - "ppl": 3.75856351852417 - }, - { - "layer": "layer.33.self_attn.q_proj", - "ppl": 3.7588233947753906 - }, - { - "layer": "layer.33.self_attn.k_proj", - "ppl": 3.7587015628814697 - }, - { - "layer": "layer.33.self_attn.v_proj", - "ppl": 3.758676767349243 - }, - { - "layer": "layer.33.self_attn.o_proj", - "ppl": 3.758779525756836 - }, - { - "layer": "layer.33.mlp.gate_proj", - "ppl": 3.758648157119751 - }, - { - "layer": "layer.33.mlp.up_proj", - "ppl": 3.7588090896606445 - }, - { - "layer": "layer.33.mlp.down_proj", - "ppl": 3.7588489055633545 - }, - { - "layer": "layer.34.self_attn.q_proj", - "ppl": 3.7587292194366455 - }, - { - "layer": "layer.34.self_attn.k_proj", - "ppl": 3.75879168510437 - }, - { - "layer": "layer.34.self_attn.v_proj", - "ppl": 3.7586541175842285 - }, - { - "layer": "layer.34.self_attn.o_proj", - "ppl": 3.7587051391601562 - }, - { - "layer": "layer.34.mlp.gate_proj", - "ppl": 3.7586898803710938 - }, - { - "layer": "layer.34.mlp.up_proj", - "ppl": 3.7587263584136963 - }, - { - "layer": "layer.34.mlp.down_proj", - "ppl": 3.758800983428955 - }, - { - "layer": "layer.35.self_attn.q_proj", - "ppl": 3.758786916732788 - }, - { - "layer": "layer.35.self_attn.k_proj", - "ppl": 3.7588109970092773 - }, - { - "layer": "layer.35.self_attn.v_proj", - "ppl": 3.758828639984131 - }, - { - "layer": "layer.35.self_attn.o_proj", - "ppl": 3.7587978839874268 - }, - { - "layer": "layer.35.mlp.gate_proj", - "ppl": 3.7587828636169434 - }, - { - "layer": "layer.35.mlp.up_proj", - "ppl": 3.7588729858398438 - }, - { - "layer": "layer.35.mlp.down_proj", - "ppl": 3.758697986602783 - }, - { - "layer": "layer.36.self_attn.q_proj", - "ppl": 3.7588284015655518 - }, - { - "layer": "layer.36.self_attn.k_proj", - "ppl": 3.7587523460388184 - }, - { - "layer": "layer.36.self_attn.v_proj", - "ppl": 3.758871078491211 - }, - { - "layer": "layer.36.self_attn.o_proj", - "ppl": 3.7587482929229736 - }, - { - "layer": "layer.36.mlp.gate_proj", - "ppl": 3.7587223052978516 - }, - { - "layer": "layer.36.mlp.up_proj", - "ppl": 3.7588019371032715 - }, - { - "layer": "layer.36.mlp.down_proj", - "ppl": 3.758692502975464 - }, - { - "layer": "layer.37.self_attn.q_proj", - "ppl": 3.758760690689087 - }, - { - "layer": "layer.37.self_attn.k_proj", - "ppl": 3.758854627609253 - }, - { - "layer": "layer.37.self_attn.v_proj", - "ppl": 3.7588284015655518 - }, - { - "layer": "layer.37.self_attn.o_proj", - "ppl": 3.75874924659729 - }, - { - "layer": "layer.37.mlp.gate_proj", - "ppl": 3.758782148361206 - }, - { - "layer": "layer.37.mlp.up_proj", - "ppl": 3.758847951889038 - }, - { - "layer": "layer.37.mlp.down_proj", - "ppl": 3.7588284015655518 - }, - { - "layer": "layer.38.self_attn.q_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.38.self_attn.k_proj", - "ppl": 3.758819103240967 - }, - { - "layer": "layer.38.self_attn.v_proj", - "ppl": 3.758824348449707 - }, - { - "layer": "layer.38.self_attn.o_proj", - "ppl": 3.758789300918579 - }, - { - "layer": "layer.38.mlp.gate_proj", - "ppl": 3.7585785388946533 - }, - { - "layer": "layer.38.mlp.up_proj", - "ppl": 3.758730173110962 - }, - { - "layer": "layer.38.mlp.down_proj", - "ppl": 3.758822441101074 - }, - { - "layer": "layer.39.self_attn.q_proj", - "ppl": 3.758664608001709 - }, - { - "layer": "layer.39.self_attn.k_proj", - "ppl": 3.758873701095581 - }, - { - "layer": "layer.39.self_attn.v_proj", - "ppl": 3.758901357650757 - }, - { - "layer": "layer.39.self_attn.o_proj", - "ppl": 3.758765935897827 - }, - { - "layer": "layer.39.mlp.gate_proj", - "ppl": 3.758687734603882 - }, - { - "layer": "layer.39.mlp.up_proj", - "ppl": 3.758845329284668 - }, - { - "layer": "layer.39.mlp.down_proj", - "ppl": 3.758924722671509 - }, - { - "layer": "layer.40.self_attn.q_proj", - "ppl": 3.7588090896606445 - }, - { - "layer": "layer.40.self_attn.k_proj", - "ppl": 3.7586991786956787 - }, - { - "layer": "layer.40.self_attn.v_proj", - "ppl": 3.7587058544158936 - }, - { - "layer": "layer.40.self_attn.o_proj", - "ppl": 3.7588913440704346 - }, - { - "layer": "layer.40.mlp.gate_proj", - "ppl": 3.7587344646453857 - }, - { - "layer": "layer.40.mlp.up_proj", - "ppl": 3.7588143348693848 - }, - { - "layer": "layer.40.mlp.down_proj", - "ppl": 3.75877046585083 - }, - { - "layer": "layer.41.self_attn.q_proj", - "ppl": 3.758824348449707 - }, - { - "layer": "layer.41.self_attn.k_proj", - "ppl": 3.75883150100708 - }, - { - "layer": "layer.41.self_attn.v_proj", - "ppl": 3.7586581707000732 - }, - { - "layer": "layer.41.self_attn.o_proj", - "ppl": 3.758695125579834 - }, - { - "layer": "layer.41.mlp.gate_proj", - "ppl": 3.758635997772217 - }, - { - "layer": "layer.41.mlp.up_proj", - "ppl": 3.7586898803710938 - }, - { - "layer": "layer.41.mlp.down_proj", - "ppl": 3.758820056915283 - }, - { - "layer": "layer.42.self_attn.q_proj", - "ppl": 3.7588295936584473 - }, - { - "layer": "layer.42.self_attn.k_proj", - "ppl": 3.758695602416992 - }, - { - "layer": "layer.42.self_attn.v_proj", - "ppl": 3.7587759494781494 - }, - { - "layer": "layer.42.self_attn.o_proj", - "ppl": 3.7587497234344482 - }, - { - "layer": "layer.42.mlp.gate_proj", - "ppl": 3.758756160736084 - }, - { - "layer": "layer.42.mlp.up_proj", - "ppl": 3.758924722671509 - }, - { - "layer": "layer.42.mlp.down_proj", - "ppl": 3.7586863040924072 - }, - { - "layer": "layer.43.self_attn.q_proj", - "ppl": 3.7587616443634033 - }, - { - "layer": "layer.43.self_attn.k_proj", - "ppl": 3.7587990760803223 - }, - { - "layer": "layer.43.self_attn.v_proj", - "ppl": 3.7587785720825195 - }, - { - "layer": "layer.43.self_attn.o_proj", - "ppl": 3.758850336074829 - }, - { - "layer": "layer.43.mlp.gate_proj", - "ppl": 3.758756160736084 - }, - { - "layer": "layer.43.mlp.up_proj", - "ppl": 3.7586863040924072 - }, - { - "layer": "layer.43.mlp.down_proj", - "ppl": 3.7587525844573975 - }, - { - "layer": "layer.44.self_attn.q_proj", - "ppl": 3.7587969303131104 - }, - { - "layer": "layer.44.self_attn.k_proj", - "ppl": 3.7588438987731934 - }, - { - "layer": "layer.44.self_attn.v_proj", - "ppl": 3.7588000297546387 - }, - { - "layer": "layer.44.self_attn.o_proj", - "ppl": 3.758680820465088 - }, - { - "layer": "layer.44.mlp.gate_proj", - "ppl": 3.758715867996216 - }, - { - "layer": "layer.44.mlp.up_proj", - "ppl": 3.7586605548858643 - }, - { - "layer": "layer.44.mlp.down_proj", - "ppl": 3.7589662075042725 - }, - { - "layer": "layer.45.self_attn.q_proj", - "ppl": 3.7588109970092773 - }, - { - "layer": "layer.45.self_attn.k_proj", - "ppl": 3.758728265762329 - }, - { - "layer": "layer.45.self_attn.v_proj", - "ppl": 3.7588679790496826 - }, - { - "layer": "layer.45.self_attn.o_proj", - "ppl": 3.758742094039917 - }, - { - "layer": "layer.45.mlp.gate_proj", - "ppl": 3.75882625579834 - }, - { - "layer": "layer.45.mlp.up_proj", - "ppl": 3.758667469024658 - }, - { - "layer": "layer.45.mlp.down_proj", - "ppl": 3.7587473392486572 - }, - { - "layer": "layer.46.self_attn.q_proj", - "ppl": 3.7588324546813965 - }, - { - "layer": "layer.46.self_attn.k_proj", - "ppl": 3.7587413787841797 - }, - { - "layer": "layer.46.self_attn.v_proj", - "ppl": 3.7587990760803223 - }, - { - "layer": "layer.46.self_attn.o_proj", - "ppl": 3.758761167526245 - }, - { - "layer": "layer.46.mlp.gate_proj", - "ppl": 3.758784770965576 - }, - { - "layer": "layer.46.mlp.up_proj", - "ppl": 3.75872802734375 - }, - { - "layer": "layer.46.mlp.down_proj", - "ppl": 3.7587571144104004 - }, - { - "layer": "layer.47.self_attn.q_proj", - "ppl": 3.7588253021240234 - }, - { - "layer": "layer.47.self_attn.k_proj", - "ppl": 3.7587625980377197 - }, - { - "layer": "layer.47.self_attn.v_proj", - "ppl": 3.758817434310913 - }, - { - "layer": "layer.47.self_attn.o_proj", - "ppl": 3.758833408355713 - }, - { - "layer": "layer.47.mlp.gate_proj", - "ppl": 3.7586705684661865 - }, - { - "layer": "layer.47.mlp.up_proj", - "ppl": 3.7587320804595947 - }, - { - "layer": "layer.47.mlp.down_proj", - "ppl": 3.7588565349578857 - }, - { - "layer": "layer.48.self_attn.q_proj", - "ppl": 3.7588043212890625 - }, - { - "layer": "layer.48.self_attn.k_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.48.self_attn.v_proj", - "ppl": 3.7588751316070557 - }, - { - "layer": "layer.48.self_attn.o_proj", - "ppl": 3.7588372230529785 - }, - { - "layer": "layer.48.mlp.gate_proj", - "ppl": 3.7587740421295166 - }, - { - "layer": "layer.48.mlp.up_proj", - "ppl": 3.758695125579834 - }, - { - "layer": "layer.48.mlp.down_proj", - "ppl": 3.7587339878082275 - }, - { - "layer": "layer.49.self_attn.q_proj", - "ppl": 3.7587554454803467 - }, - { - "layer": "layer.49.self_attn.k_proj", - "ppl": 3.7587709426879883 - }, - { - "layer": "layer.49.self_attn.v_proj", - "ppl": 3.7587127685546875 - }, - { - "layer": "layer.49.self_attn.o_proj", - "ppl": 3.75882625579834 - }, - { - "layer": "layer.49.mlp.gate_proj", - "ppl": 3.758580207824707 - }, - { - "layer": "layer.49.mlp.up_proj", - "ppl": 3.7588512897491455 - }, - { - "layer": "layer.49.mlp.down_proj", - "ppl": 3.7586677074432373 - }, - { - "layer": "layer.50.self_attn.q_proj", - "ppl": 3.758765697479248 - }, - { - "layer": "layer.50.self_attn.k_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.50.self_attn.v_proj", - "ppl": 3.7588376998901367 - }, - { - "layer": "layer.50.self_attn.o_proj", - "ppl": 3.758859395980835 - }, - { - "layer": "layer.50.mlp.gate_proj", - "ppl": 3.7586405277252197 - }, - { - "layer": "layer.50.mlp.up_proj", - "ppl": 3.7586827278137207 - }, - { - "layer": "layer.50.mlp.down_proj", - "ppl": 3.7587554454803467 - }, - { - "layer": "layer.51.self_attn.q_proj", - "ppl": 3.7588319778442383 - }, - { - "layer": "layer.51.self_attn.k_proj", - "ppl": 3.7588162422180176 - }, - { - "layer": "layer.51.self_attn.v_proj", - "ppl": 3.7587106227874756 - }, - { - "layer": "layer.51.self_attn.o_proj", - "ppl": 3.7588648796081543 - }, - { - "layer": "layer.51.mlp.gate_proj", - "ppl": 3.7587273120880127 - }, - { - "layer": "layer.51.mlp.up_proj", - "ppl": 3.7590277194976807 - }, - { - "layer": "layer.51.mlp.down_proj", - "ppl": 3.7587673664093018 - }, - { - "layer": "layer.52.self_attn.q_proj", - "ppl": 3.7587811946868896 - }, - { - "layer": "layer.52.self_attn.k_proj", - "ppl": 3.7587485313415527 - }, - { - "layer": "layer.52.self_attn.v_proj", - "ppl": 3.7587733268737793 - }, - { - "layer": "layer.52.self_attn.o_proj", - "ppl": 3.758805274963379 - }, - { - "layer": "layer.52.mlp.gate_proj", - "ppl": 3.758598804473877 - }, - { - "layer": "layer.52.mlp.up_proj", - "ppl": 3.758692502975464 - }, - { - "layer": "layer.52.mlp.down_proj", - "ppl": 3.7586469650268555 - }, - { - "layer": "layer.53.self_attn.q_proj", - "ppl": 3.7587971687316895 - }, - { - "layer": "layer.53.self_attn.k_proj", - "ppl": 3.7588090896606445 - }, - { - "layer": "layer.53.self_attn.v_proj", - "ppl": 3.758812427520752 - }, - { - "layer": "layer.53.self_attn.o_proj", - "ppl": 3.758866548538208 - }, - { - "layer": "layer.53.mlp.gate_proj", - "ppl": 3.7586798667907715 - }, - { - "layer": "layer.53.mlp.up_proj", - "ppl": 3.758822202682495 - }, - { - "layer": "layer.53.mlp.down_proj", - "ppl": 3.7587897777557373 - }, - { - "layer": "layer.54.self_attn.q_proj", - "ppl": 3.7587740421295166 - }, - { - "layer": "layer.54.self_attn.k_proj", - "ppl": 3.7587811946868896 - }, - { - "layer": "layer.54.self_attn.v_proj", - "ppl": 3.758793830871582 - }, - { - "layer": "layer.54.self_attn.o_proj", - "ppl": 3.758777141571045 - }, - { - "layer": "layer.54.mlp.gate_proj", - "ppl": 3.758711338043213 - }, - { - "layer": "layer.54.mlp.up_proj", - "ppl": 3.7586827278137207 - }, - { - "layer": "layer.54.mlp.down_proj", - "ppl": 3.758742094039917 - }, - { - "layer": "layer.55.self_attn.q_proj", - "ppl": 3.75881028175354 - }, - { - "layer": "layer.55.self_attn.k_proj", - "ppl": 3.758760690689087 - }, - { - "layer": "layer.55.self_attn.v_proj", - "ppl": 3.758746385574341 - }, - { - "layer": "layer.55.self_attn.o_proj", - "ppl": 3.758786678314209 - }, - { - "layer": "layer.55.mlp.gate_proj", - "ppl": 3.758516788482666 - }, - { - "layer": "layer.55.mlp.up_proj", - "ppl": 3.75864315032959 - }, - { - "layer": "layer.55.mlp.down_proj", - "ppl": 3.7587804794311523 - }, - { - "layer": "layer.56.self_attn.q_proj", - "ppl": 3.7587852478027344 - }, - { - "layer": "layer.56.self_attn.k_proj", - "ppl": 3.758756160736084 - }, - { - "layer": "layer.56.self_attn.v_proj", - "ppl": 3.7587249279022217 - }, - { - "layer": "layer.56.self_attn.o_proj", - "ppl": 3.7587530612945557 - }, - { - "layer": "layer.56.mlp.gate_proj", - "ppl": 3.758800983428955 - }, - { - "layer": "layer.56.mlp.up_proj", - "ppl": 3.7587451934814453 - }, - { - "layer": "layer.56.mlp.down_proj", - "ppl": 3.758817195892334 - }, - { - "layer": "layer.57.self_attn.q_proj", - "ppl": 3.7588324546813965 - }, - { - "layer": "layer.57.self_attn.k_proj", - "ppl": 3.7588679790496826 - }, - { - "layer": "layer.57.self_attn.v_proj", - "ppl": 3.7587764263153076 - }, - { - "layer": "layer.57.self_attn.o_proj", - "ppl": 3.75874924659729 - }, - { - "layer": "layer.57.mlp.gate_proj", - "ppl": 3.7587900161743164 - }, - { - "layer": "layer.57.mlp.up_proj", - "ppl": 3.7588119506835938 - }, - { - "layer": "layer.57.mlp.down_proj", - "ppl": 3.7587530612945557 - }, - { - "layer": "layer.58.self_attn.q_proj", - "ppl": 3.75876784324646 - }, - { - "layer": "layer.58.self_attn.k_proj", - "ppl": 3.758708953857422 - }, - { - "layer": "layer.58.self_attn.v_proj", - "ppl": 3.7587435245513916 - }, - { - "layer": "layer.58.self_attn.o_proj", - "ppl": 3.758777618408203 - }, - { - "layer": "layer.58.mlp.gate_proj", - "ppl": 3.758777618408203 - }, - { - "layer": "layer.58.mlp.up_proj", - "ppl": 3.758652448654175 - }, - { - "layer": "layer.58.mlp.down_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.59.self_attn.q_proj", - "ppl": 3.7587828636169434 - }, - { - "layer": "layer.59.self_attn.k_proj", - "ppl": 3.7587714195251465 - }, - { - "layer": "layer.59.self_attn.v_proj", - "ppl": 3.758822441101074 - }, - { - "layer": "layer.59.self_attn.o_proj", - "ppl": 3.758753538131714 - }, - { - "layer": "layer.59.mlp.gate_proj", - "ppl": 3.7587804794311523 - }, - { - "layer": "layer.59.mlp.up_proj", - "ppl": 3.75876522064209 - }, - { - "layer": "layer.59.mlp.down_proj", - "ppl": 3.758790969848633 - }, - { - "layer": "layer.60.self_attn.q_proj", - "ppl": 3.7587785720825195 - }, - { - "layer": "layer.60.self_attn.k_proj", - "ppl": 3.75877046585083 - }, - { - "layer": "layer.60.self_attn.v_proj", - "ppl": 3.7588493824005127 - }, - { - "layer": "layer.60.self_attn.o_proj", - "ppl": 3.7587714195251465 - }, - { - "layer": "layer.60.mlp.gate_proj", - "ppl": 3.7586169242858887 - }, - { - "layer": "layer.60.mlp.up_proj", - "ppl": 3.7587642669677734 - }, - { - "layer": "layer.60.mlp.down_proj", - "ppl": 3.758525848388672 - }, - { - "layer": "layer.61.self_attn.q_proj", - "ppl": 3.7587482929229736 - }, - { - "layer": "layer.61.self_attn.k_proj", - "ppl": 3.758788824081421 - }, - { - "layer": "layer.61.self_attn.v_proj", - "ppl": 3.758859395980835 - }, - { - "layer": "layer.61.self_attn.o_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.61.mlp.gate_proj", - "ppl": 3.7586209774017334 - }, - { - "layer": "layer.61.mlp.up_proj", - "ppl": 3.758772373199463 - }, - { - "layer": "layer.61.mlp.down_proj", - "ppl": 3.758589744567871 - }, - { - "layer": "layer.62.self_attn.q_proj", - "ppl": 3.758807897567749 - }, - { - "layer": "layer.62.self_attn.k_proj", - "ppl": 3.7587969303131104 - }, - { - "layer": "layer.62.self_attn.v_proj", - "ppl": 3.7587997913360596 - }, - { - "layer": "layer.62.self_attn.o_proj", - "ppl": 3.758774995803833 - }, - { - "layer": "layer.62.mlp.gate_proj", - "ppl": 3.7587947845458984 - }, - { - "layer": "layer.62.mlp.up_proj", - "ppl": 3.7588376998901367 - }, - { - "layer": "layer.62.mlp.down_proj", - "ppl": 3.7586119174957275 - }, - { - "layer": "layer.63.self_attn.q_proj", - "ppl": 3.7588143348693848 - }, - { - "layer": "layer.63.self_attn.k_proj", - "ppl": 3.7587828636169434 - }, - { - "layer": "layer.63.self_attn.v_proj", - "ppl": 3.7588043212890625 - }, - { - "layer": "layer.63.self_attn.o_proj", - "ppl": 3.758819341659546 - }, - { - "layer": "layer.63.mlp.gate_proj", - "ppl": 3.7585713863372803 - }, - { - "layer": "layer.63.mlp.up_proj", - "ppl": 3.7587966918945312 - }, - { - "layer": "layer.63.mlp.down_proj", - "ppl": 3.7587413787841797 - }, - { - "layer": "layer.64.self_attn.q_proj", - "ppl": 3.758812427520752 - }, - { - "layer": "layer.64.self_attn.k_proj", - "ppl": 3.7587714195251465 - }, - { - "layer": "layer.64.self_attn.v_proj", - "ppl": 3.7588043212890625 - }, - { - "layer": "layer.64.self_attn.o_proj", - "ppl": 3.758781671524048 - }, - { - "layer": "layer.64.mlp.gate_proj", - "ppl": 3.758732318878174 - }, - { - "layer": "layer.64.mlp.up_proj", - "ppl": 3.7587547302246094 - }, - { - "layer": "layer.64.mlp.down_proj", - "ppl": 3.7588133811950684 - }, - { - "layer": "layer.65.self_attn.q_proj", - "ppl": 3.758838176727295 - }, - { - "layer": "layer.65.self_attn.k_proj", - "ppl": 3.7587473392486572 - }, - { - "layer": "layer.65.self_attn.v_proj", - "ppl": 3.7587971687316895 - }, - { - "layer": "layer.65.self_attn.o_proj", - "ppl": 3.758774995803833 - }, - { - "layer": "layer.65.mlp.gate_proj", - "ppl": 3.7587263584136963 - }, - { - "layer": "layer.65.mlp.up_proj", - "ppl": 3.758695125579834 - }, - { - "layer": "layer.65.mlp.down_proj", - "ppl": 3.7589356899261475 - }, - { - "layer": "layer.66.self_attn.q_proj", - "ppl": 3.7588162422180176 - }, - { - "layer": "layer.66.self_attn.k_proj", - "ppl": 3.758782148361206 - }, - { - "layer": "layer.66.self_attn.v_proj", - "ppl": 3.7587971687316895 - }, - { - "layer": "layer.66.self_attn.o_proj", - "ppl": 3.758840560913086 - }, - { - "layer": "layer.66.mlp.gate_proj", - "ppl": 3.758479118347168 - }, - { - "layer": "layer.66.mlp.up_proj", - "ppl": 3.758807897567749 - }, - { - "layer": "layer.66.mlp.down_proj", - "ppl": 3.758709669113159 - }, - { - "layer": "layer.67.self_attn.q_proj", - "ppl": 3.7587673664093018 - }, - { - "layer": "layer.67.self_attn.k_proj", - "ppl": 3.758777141571045 - }, - { - "layer": "layer.67.self_attn.v_proj", - "ppl": 3.758819103240967 - }, - { - "layer": "layer.67.self_attn.o_proj", - "ppl": 3.7588119506835938 - }, - { - "layer": "layer.67.mlp.gate_proj", - "ppl": 3.7586894035339355 - }, - { - "layer": "layer.67.mlp.up_proj", - "ppl": 3.7588043212890625 - }, - { - "layer": "layer.67.mlp.down_proj", - "ppl": 3.7586827278137207 - }, - { - "layer": "layer.68.self_attn.q_proj", - "ppl": 3.75877046585083 - }, - { - "layer": "layer.68.self_attn.k_proj", - "ppl": 3.758817434310913 - }, - { - "layer": "layer.68.self_attn.v_proj", - "ppl": 3.758784532546997 - }, - { - "layer": "layer.68.self_attn.o_proj", - "ppl": 3.7588448524475098 - }, - { - "layer": "layer.68.mlp.gate_proj", - "ppl": 3.7586565017700195 - }, - { - "layer": "layer.68.mlp.up_proj", - "ppl": 3.758920192718506 - }, - { - "layer": "layer.68.mlp.down_proj", - "ppl": 3.758819103240967 - }, - { - "layer": "layer.69.self_attn.q_proj", - "ppl": 3.758852958679199 - }, - { - "layer": "layer.69.self_attn.k_proj", - "ppl": 3.758842706680298 - }, - { - "layer": "layer.69.self_attn.v_proj", - "ppl": 3.7587811946868896 - }, - { - "layer": "layer.69.self_attn.o_proj", - "ppl": 3.7588627338409424 - }, - { - "layer": "layer.69.mlp.gate_proj", - "ppl": 3.758753538131714 - }, - { - "layer": "layer.69.mlp.up_proj", - "ppl": 3.7585761547088623 - }, - { - "layer": "layer.69.mlp.down_proj", - "ppl": 3.758662223815918 - }, - { - "layer": "layer.70.self_attn.q_proj", - "ppl": 3.7587759494781494 - }, - { - "layer": "layer.70.self_attn.k_proj", - "ppl": 3.758763313293457 - }, - { - "layer": "layer.70.self_attn.v_proj", - "ppl": 3.7587313652038574 - }, - { - "layer": "layer.70.self_attn.o_proj", - "ppl": 3.7588698863983154 - }, - { - "layer": "layer.70.mlp.gate_proj", - "ppl": 3.7586944103240967 - }, - { - "layer": "layer.70.mlp.up_proj", - "ppl": 3.7588071823120117 - }, - { - "layer": "layer.70.mlp.down_proj", - "ppl": 3.7588467597961426 - }, - { - "layer": "layer.71.self_attn.q_proj", - "ppl": 3.7587759494781494 - }, - { - "layer": "layer.71.self_attn.k_proj", - "ppl": 3.7587764263153076 - }, - { - "layer": "layer.71.self_attn.v_proj", - "ppl": 3.758737325668335 - }, - { - "layer": "layer.71.self_attn.o_proj", - "ppl": 3.7587571144104004 - }, - { - "layer": "layer.71.mlp.gate_proj", - "ppl": 3.758532762527466 - }, - { - "layer": "layer.71.mlp.up_proj", - "ppl": 3.7587454319000244 - }, - { - "layer": "layer.71.mlp.down_proj", - "ppl": 3.7588746547698975 - }, - { - "layer": "layer.72.self_attn.q_proj", - "ppl": 3.7587320804595947 - }, - { - "layer": "layer.72.self_attn.k_proj", - "ppl": 3.7587900161743164 - }, - { - "layer": "layer.72.self_attn.v_proj", - "ppl": 3.7588601112365723 - }, - { - "layer": "layer.72.self_attn.o_proj", - "ppl": 3.7587578296661377 - }, - { - "layer": "layer.72.mlp.gate_proj", - "ppl": 3.7587544918060303 - }, - { - "layer": "layer.72.mlp.up_proj", - "ppl": 3.7589023113250732 - }, - { - "layer": "layer.72.mlp.down_proj", - "ppl": 3.758735418319702 - }, - { - "layer": "layer.73.self_attn.q_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.73.self_attn.k_proj", - "ppl": 3.758798122406006 - }, - { - "layer": "layer.73.self_attn.v_proj", - "ppl": 3.75874924659729 - }, - { - "layer": "layer.73.self_attn.o_proj", - "ppl": 3.758800983428955 - }, - { - "layer": "layer.73.mlp.gate_proj", - "ppl": 3.758641242980957 - }, - { - "layer": "layer.73.mlp.up_proj", - "ppl": 3.75876784324646 - }, - { - "layer": "layer.73.mlp.down_proj", - "ppl": 3.758824348449707 - }, - { - "layer": "layer.74.self_attn.q_proj", - "ppl": 3.758819341659546 - }, - { - "layer": "layer.74.self_attn.k_proj", - "ppl": 3.758824348449707 - }, - { - "layer": "layer.74.self_attn.v_proj", - "ppl": 3.758726119995117 - }, - { - "layer": "layer.74.self_attn.o_proj", - "ppl": 3.758843183517456 - }, - { - "layer": "layer.74.mlp.gate_proj", - "ppl": 3.7585527896881104 - }, - { - "layer": "layer.74.mlp.up_proj", - "ppl": 3.7588584423065186 - }, - { - "layer": "layer.74.mlp.down_proj", - "ppl": 3.7587966918945312 - }, - { - "layer": "layer.75.self_attn.q_proj", - "ppl": 3.7587785720825195 - }, - { - "layer": "layer.75.self_attn.k_proj", - "ppl": 3.7587759494781494 - }, - { - "layer": "layer.75.self_attn.v_proj", - "ppl": 3.758760690689087 - }, - { - "layer": "layer.75.self_attn.o_proj", - "ppl": 3.758725881576538 - }, - { - "layer": "layer.75.mlp.gate_proj", - "ppl": 3.75856614112854 - }, - { - "layer": "layer.75.mlp.up_proj", - "ppl": 3.7586429119110107 - }, - { - "layer": "layer.75.mlp.down_proj", - "ppl": 3.758655071258545 - }, - { - "layer": "layer.76.self_attn.q_proj", - "ppl": 3.7588882446289062 - }, - { - "layer": "layer.76.self_attn.k_proj", - "ppl": 3.758915662765503 - }, - { - "layer": "layer.76.self_attn.v_proj", - "ppl": 3.758843183517456 - }, - { - "layer": "layer.76.self_attn.o_proj", - "ppl": 3.7587687969207764 - }, - { - "layer": "layer.76.mlp.gate_proj", - "ppl": 3.758679151535034 - }, - { - "layer": "layer.76.mlp.up_proj", - "ppl": 3.7588231563568115 - }, - { - "layer": "layer.76.mlp.down_proj", - "ppl": 3.758749485015869 - }, - { - "layer": "layer.77.self_attn.q_proj", - "ppl": 3.758755683898926 - }, - { - "layer": "layer.77.self_attn.k_proj", - "ppl": 3.758852958679199 - }, - { - "layer": "layer.77.self_attn.v_proj", - "ppl": 3.7588367462158203 - }, - { - "layer": "layer.77.self_attn.o_proj", - "ppl": 3.758790969848633 - }, - { - "layer": "layer.77.mlp.gate_proj", - "ppl": 3.7586307525634766 - }, - { - "layer": "layer.77.mlp.up_proj", - "ppl": 3.7587432861328125 - }, - { - "layer": "layer.77.mlp.down_proj", - "ppl": 3.758852005004883 - }, - { - "layer": "layer.78.self_attn.q_proj", - "ppl": 3.7587497234344482 - }, - { - "layer": "layer.78.self_attn.k_proj", - "ppl": 3.758779525756836 - }, - { - "layer": "layer.78.self_attn.v_proj", - "ppl": 3.7587339878082275 - }, - { - "layer": "layer.78.self_attn.o_proj", - "ppl": 3.758721113204956 - }, - { - "layer": "layer.78.mlp.gate_proj", - "ppl": 3.7585947513580322 - }, - { - "layer": "layer.78.mlp.up_proj", - "ppl": 3.758655071258545 - }, - { - "layer": "layer.78.mlp.down_proj", - "ppl": 3.7589023113250732 - }, - { - "layer": "layer.79.self_attn.q_proj", - "ppl": 3.7588584423065186 - }, - { - "layer": "layer.79.self_attn.k_proj", - "ppl": 3.758791923522949 - }, - { - "layer": "layer.79.self_attn.v_proj", - "ppl": 3.7587263584136963 - }, - { - "layer": "layer.79.self_attn.o_proj", - "ppl": 3.7587525844573975 - }, - { - "layer": "layer.79.mlp.gate_proj", - "ppl": 3.7585508823394775 - }, - { - "layer": "layer.79.mlp.up_proj", - "ppl": 3.758737325668335 - }, - { - "layer": "layer.79.mlp.down_proj", - "ppl": 3.758657455444336 - } - ] - } - ] -} \ No newline at end of file