diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,20240 @@ +{ + "original_ppl": 5.512016773223877, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 68466.640625 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 68821.2578125 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 55653.11328125 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 315082.3125 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 56351.7890625 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 59148.0390625 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 57236.140625 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 69829.140625 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 69498.953125 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 103642.546875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 105510.375 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 67349.578125 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 64738.15234375 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 56554.00390625 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 69732.3828125 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 69418.3359375 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 75194.8828125 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 73319.1796875 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 62787.19140625 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 58990.421875 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 63073.4609375 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 69344.2890625 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 69535.6796875 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 77953.5390625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 84253.8828125 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 66028.84375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 77675.5546875 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 97159.5625 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 69991.4140625 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 69781.34375 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 60276.90625 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 49056.3984375 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 91540.75 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 58227.203125 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 78319.1171875 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 74254.0859375 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 70581.953125 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 68964.9453125 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 146452.140625 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 72120.0 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 71005.1328125 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 69268.9453125 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 72182.6171875 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 68823.484375 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 77006.9375 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 100681.9765625 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 73224.4296875 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 88270.875 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 95050.4453125 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 70094.5546875 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 68899.671875 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 68353.5078125 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 62955.4375 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 66862.8203125 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 79292.671875 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 84155.0234375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 66311.109375 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 69777.0859375 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 86166.6484375 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 88211.546875 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 72616.421875 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 69410.390625 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 84706.5859375 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 72150.0546875 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 70723.3203125 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 74035.5234375 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 86030.015625 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 73489.71875 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 77266.8359375 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 76658.5234375 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 70742.0078125 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 68781.8203125 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 89308.1484375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 58563.79296875 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 60792.3203125 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 74782.6796875 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 70480.1171875 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 69279.578125 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 70762.71875 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 81226.78125 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 78539.7734375 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 74164.4921875 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 107293.6171875 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 70990.1015625 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 66809.3359375 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 70804.5703125 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 62976.93359375 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 44584.48046875 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 71926.984375 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 67561.484375 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 64757.2890625 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 70122.90625 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 70287.609375 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 83639.765625 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 71206.265625 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 63999.609375 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 71756.1796875 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 62125.15625 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 68458.0234375 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 70032.546875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 55534.66796875 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 70755.96875 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 81528.5234375 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 72866.09375 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 71369.6328125 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 73363.53125 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 68266.21875 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 68733.6875 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 46430.9609375 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 67492.3203125 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 72795.0390625 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 65192.09765625 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 70056.9921875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 69781.4140625 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 60702.98828125 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 63565.60546875 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 89711.828125 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 72218.5546875 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 59242.54296875 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 70826.390625 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 72487.3828125 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 82313.8359375 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 74436.7265625 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 69760.71875 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 70996.9375 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 61572.77734375 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 68727.921875 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 67984.8984375 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 56699.48828125 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 58351.61328125 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 69728.2578125 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 65963.578125 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 75159.7578125 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 68513.5390625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 68453.3203125 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 70702.75 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 64727.78125 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 76043.6171875 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 65647.28125 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 70076.0390625 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 71369.21875 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 70260.1328125 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 73725.015625 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 50186.03125 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 72479.2265625 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 60537.93359375 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 84326.390625 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 71414.90625 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 69883.1015625 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 54566.16796875 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 63556.02734375 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 84233.5546875 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 74697.5 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 74207.15625 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 69079.3515625 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 68429.1015625 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 88330.25 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 60508.78515625 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 63157.85546875 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 63940.25 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 69267.8203125 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 69816.5625 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 70709.828125 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 57969.61328125 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 59764.26953125 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 67366.921875 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 76275.234375 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 79164.515625 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 68558.765625 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 69455.6171875 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 60702.46484375 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 62240.26171875 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 70316.375 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 74913.5859375 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 68145.1015625 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 70109.3984375 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 67956.1171875 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 67313.296875 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 52667.29296875 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 65196.51171875 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 85032.109375 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 79209.984375 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 68986.2578125 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 71939.609375 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 48879.78125 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 79738.1015625 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 70545.078125 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 76305.8671875 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 72321.390625 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 69885.765625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 69538.796875 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 65936.34375 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 66677.9609375 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 87288.3359375 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 69464.234375 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 65497.515625 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 70092.6796875 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 69784.8046875 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 62454.19140625 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 47236.1015625 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 69493.6484375 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 63683.44140625 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 61081.78515625 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 70129.859375 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 69277.796875 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 53455.77734375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 83844.53125 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 68565.171875 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 73007.640625 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 71513.6640625 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 69608.6640625 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 68884.9453125 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 81813.7421875 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 59919.95703125 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 69867.5078125 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 68889.875 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 64510.85546875 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 70602.9609375 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 69793.328125 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 77319.171875 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 70541.0390625 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 71421.78125 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 73810.984375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 74824.1953125 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 69145.390625 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 68929.5703125 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 74096.5546875 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 72532.5390625 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 73568.953125 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 68509.09375 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 72827.8828125 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 69336.421875 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 69925.3671875 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 74330.53125 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 66528.9453125 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 67342.3828125 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 66030.484375 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 70191.015625 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 70147.515625 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 69621.609375 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 73055.9765625 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 76648.578125 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 66215.625 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 68045.3515625 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 67854.5703125 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 70492.28125 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 68449.53125 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 61326.11328125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 57157.05078125 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 67152.6171875 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 70807.5390625 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 63696.0703125 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 69476.15625 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 70141.765625 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 61875.578125 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 71030.4609375 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 61146.01171875 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 66463.6953125 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 72927.6875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 69252.6953125 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 69661.5234375 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 75972.71875 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 72677.8125 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 71059.9375 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 67001.96875 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 66510.046875 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 69901.3671875 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 69892.03125 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 71998.296875 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 79787.2421875 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 68233.28125 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 65367.34765625 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 71320.5703125 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 69722.0078125 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 69932.0390625 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 62778.92578125 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 76499.0234375 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 70142.4296875 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 84644.9765625 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 72061.4921875 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 70005.171875 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 69819.8203125 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 65251.74609375 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 63913.55078125 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 66499.0078125 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 68097.734375 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 69707.3828125 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 70056.796875 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 69660.734375 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 76951.4375 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 88786.8046875 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 75250.703125 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 69283.7421875 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 67995.3984375 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 69721.875 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 69813.03125 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 76007.8671875 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 72060.9453125 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 70513.46875 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 70337.3671875 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 69666.375 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 69642.9296875 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 69014.8203125 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 87117.8515625 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 64734.9375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 73587.484375 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 69809.4375 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 70834.421875 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 69729.921875 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 69775.0859375 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 71842.53125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 79723.9609375 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 76784.515625 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 66542.71875 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 66499.1328125 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 69520.03125 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 69532.296875 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 90502.0625 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 75543.640625 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 63808.55078125 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 71195.9453125 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 67488.265625 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 70733.4375 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 68811.28125 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 82223.6171875 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 73948.375 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 72397.7109375 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 64257.87890625 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 64340.90625 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 69803.7109375 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 69938.5703125 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 69962.1171875 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 68595.0 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 78923.1484375 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 64808.13671875 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 61563.49609375 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 152.70852661132812 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 156.61598205566406 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 123.35676574707031 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 125.5961685180664 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 133.86827087402344 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 135.96194458007812 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 139.04747009277344 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 168.97958374023438 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 169.33892822265625 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 160.88201904296875 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 188.765625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 163.4228057861328 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 160.48512268066406 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 150.7383270263672 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 168.47706604003906 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 167.99459838867188 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 170.95025634765625 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 156.5911865234375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 171.7934112548828 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 164.96823120117188 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 159.80245971679688 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 168.8822784423828 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 168.6259002685547 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 171.11851501464844 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 164.60223388671875 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 153.6232147216797 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 188.7719268798828 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 175.9442138671875 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 168.42694091796875 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 168.41119384765625 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 184.8650360107422 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 174.62445068359375 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 184.41433715820312 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 181.626953125 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 171.83847045898438 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 168.81320190429688 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 165.53456115722656 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 153.86427307128906 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 152.7667999267578 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 133.68763732910156 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 152.9687042236328 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 164.47010803222656 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 158.08468627929688 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 164.73602294921875 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 163.12094116210938 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 165.02645874023438 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 151.993408203125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 162.02975463867188 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 158.0990753173828 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 166.27098083496094 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 167.53501892089844 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 160.32122802734375 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 162.58009338378906 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 155.87242126464844 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 167.12062072753906 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 164.85562133789062 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 168.93946838378906 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 170.6643829345703 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 166.45819091796875 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 172.48883056640625 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 162.16726684570312 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 161.5653839111328 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 168.7418212890625 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 167.05813598632812 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 168.23806762695312 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 170.43780517578125 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 166.04690551757812 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 160.31381225585938 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 166.0361328125 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 162.96995544433594 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 167.8301544189453 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 167.28652954101562 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 175.0531463623047 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 173.47525024414062 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 163.34432983398438 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 165.61407470703125 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 167.8603973388672 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 168.5032501220703 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 168.51161193847656 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 166.65109252929688 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 168.04844665527344 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 166.43597412109375 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 167.20199584960938 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 166.42921447753906 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 171.25770568847656 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 168.84645080566406 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 168.35281372070312 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 169.56590270996094 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 164.5697479248047 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 167.688720703125 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 163.89047241210938 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 167.99917602539062 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 169.13758850097656 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 169.74703979492188 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 167.4007110595703 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 165.4798583984375 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 167.0482635498047 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 166.8798828125 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 169.65121459960938 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 169.67654418945312 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 168.5060577392578 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 166.3931121826172 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 166.73687744140625 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 167.2801513671875 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 167.42123413085938 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 168.44798278808594 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 167.9149932861328 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 169.42147827148438 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 166.06622314453125 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 165.7342987060547 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 169.1624298095703 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 168.7198486328125 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 169.9687042236328 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 169.16615295410156 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 168.95968627929688 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 168.08291625976562 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 164.58480834960938 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 168.3854217529297 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 168.97056579589844 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 168.37498474121094 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 168.8004913330078 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 164.6213836669922 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 162.23191833496094 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 165.8158721923828 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 168.48460388183594 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 168.84507751464844 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 168.29110717773438 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 169.4351348876953 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 168.6822052001953 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 171.56280517578125 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 166.64459228515625 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 166.27651977539062 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 168.86505126953125 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 168.17701721191406 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 169.0776824951172 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 166.72479248046875 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 172.46670532226562 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 167.5477294921875 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 167.1471405029297 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 167.12171936035156 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 169.71214294433594 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 168.80128479003906 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 169.01890563964844 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 164.00633239746094 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 167.23092651367188 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 166.3931121826172 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 168.80514526367188 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 168.06912231445312 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 168.65695190429688 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 168.23533630371094 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 162.53273010253906 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 165.7168426513672 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 168.1372528076172 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 170.495361328125 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 168.41343688964844 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 168.49842834472656 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 166.95811462402344 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 163.54119873046875 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 167.2679443359375 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 167.7006378173828 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 168.15472412109375 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 167.9990234375 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 168.2654266357422 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 173.18035888671875 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 168.3035430908203 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 164.2694549560547 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 165.89724731445312 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 166.83181762695312 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 168.190185546875 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 169.21632385253906 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 168.35546875 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 165.55540466308594 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 166.33250427246094 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 167.2493438720703 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 168.96653747558594 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 168.67384338378906 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 168.73497009277344 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 167.23284912109375 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 167.34852600097656 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 165.87977600097656 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 167.79444885253906 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 167.6543426513672 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 167.94046020507812 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 168.98345947265625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 167.7516632080078 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 167.41763305664062 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 166.03091430664062 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 169.77479553222656 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 169.01116943359375 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 167.15438842773438 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 168.10423278808594 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 165.86727905273438 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 167.8754425048828 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 167.4071807861328 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 166.46438598632812 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 168.05020141601562 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 168.29855346679688 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 167.39073181152344 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 168.35064697265625 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 171.13075256347656 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 166.39271545410156 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 166.75125122070312 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 167.8343048095703 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 169.00634765625 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 168.1868133544922 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 166.7734375 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 168.0054931640625 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 168.08627319335938 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 167.07113647460938 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 167.59512329101562 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 168.97145080566406 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 168.54141235351562 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 167.06619262695312 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 166.69076538085938 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 167.6554718017578 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 168.2853240966797 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 167.2333984375 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 168.54246520996094 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 168.39271545410156 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 170.0218048095703 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 167.94302368164062 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 166.90916442871094 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 166.4524688720703 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 166.6334686279297 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 168.63040161132812 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 168.5122528076172 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 169.80523681640625 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 166.5950927734375 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 166.95883178710938 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 168.85040283203125 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 167.8460693359375 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 168.7130126953125 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 168.5835418701172 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 168.14920043945312 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 168.11578369140625 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 168.11985778808594 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 165.65670776367188 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 165.30747985839844 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 168.96121215820312 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 168.15264892578125 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 165.7439422607422 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 166.53607177734375 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 167.33160400390625 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 167.92019653320312 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 168.2858123779297 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 168.44146728515625 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 167.97161865234375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 168.42404174804688 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 166.28199768066406 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 166.718505859375 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 167.04977416992188 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 166.48074340820312 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 168.55026245117188 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 168.81472778320312 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 168.22048950195312 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 167.9075469970703 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 166.8743133544922 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 166.88409423828125 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 167.12020874023438 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 168.1271514892578 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 168.24961853027344 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 167.46505737304688 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 167.44517517089844 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 167.19146728515625 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 167.26722717285156 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 167.47727966308594 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 168.77609252929688 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 168.63040161132812 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 168.16436767578125 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 168.31356811523438 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 167.71414184570312 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 168.67962646484375 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 167.6998291015625 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 168.48300170898438 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 168.59857177734375 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 169.09420776367188 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 167.9806671142578 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 166.64974975585938 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 168.9393768310547 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 167.49452209472656 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 168.84927368164062 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 168.6447296142578 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 167.73678588867188 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 169.072265625 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 167.73814392089844 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 168.86456298828125 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 167.83135986328125 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 168.57574462890625 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 168.2803497314453 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 167.54110717773438 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 169.15509033203125 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 167.253173828125 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 168.72227478027344 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 168.77423095703125 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 168.77230834960938 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 168.6370849609375 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 168.9281005859375 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 169.94845581054688 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 167.8938751220703 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 168.85191345214844 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 168.44195556640625 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 168.12796020507812 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 168.45489501953125 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 167.05638122558594 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 166.5954132080078 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 168.15023803710938 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 168.6923370361328 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 168.03033447265625 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 168.5831298828125 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 168.08033752441406 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 169.503173828125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 168.70777893066406 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 168.45054626464844 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 168.0343475341797 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 168.4623565673828 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 168.3980255126953 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 168.3740997314453 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 168.824462890625 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 168.9929656982422 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 166.97825622558594 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 169.83309936523438 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 168.27857971191406 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 168.3643798828125 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 168.36880493164062 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 170.42724609375 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 168.90853881835938 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 166.77113342285156 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 167.31204223632812 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 164.26937866210938 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 168.67214965820312 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 169.1435546875 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 168.70465087890625 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 168.58482360839844 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 164.01368713378906 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 164.74734497070312 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 163.9591064453125 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 8.789073944091797 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 8.80109977722168 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 8.814238548278809 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 8.953442573547363 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 8.771869659423828 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 8.756406784057617 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 8.746805191040039 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 8.808968544006348 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 8.811210632324219 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 8.77315616607666 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 8.834480285644531 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 8.782471656799316 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 8.752105712890625 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 8.736794471740723 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 8.816064834594727 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 8.811182022094727 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 8.804661750793457 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 8.865989685058594 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 8.819738388061523 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 8.843111991882324 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 8.794148445129395 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 8.827264785766602 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 8.808921813964844 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 8.770608901977539 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 8.817179679870605 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 8.800025939941406 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 8.896750450134277 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 8.894807815551758 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 8.814414024353027 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 8.820975303649902 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 8.773513793945312 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 8.836915969848633 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 8.833911895751953 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 8.821717262268066 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 8.768404960632324 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 8.83438777923584 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 8.828968048095703 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 8.993260383605957 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 8.723670959472656 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 8.767157554626465 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 8.752811431884766 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 8.774773597717285 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 8.717400550842285 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 8.778027534484863 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 8.734466552734375 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 8.765472412109375 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 8.716459274291992 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 8.766624450683594 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 8.798978805541992 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 8.808492660522461 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 8.807854652404785 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 8.806876182556152 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 8.792720794677734 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 8.780024528503418 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 8.804308891296387 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 8.795166969299316 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 8.825348854064941 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 8.814411163330078 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 8.814659118652344 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 8.821195602416992 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 8.785569190979004 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 8.803384780883789 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 8.806739807128906 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 8.815083503723145 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 8.81592845916748 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 8.80963134765625 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 8.80908489227295 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 8.800393104553223 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 8.795150756835938 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 8.800907135009766 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 8.811037063598633 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 8.809061050415039 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 8.818999290466309 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 8.80842399597168 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 8.801939010620117 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 8.805944442749023 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 8.804067611694336 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 8.808344841003418 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 8.81078052520752 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 8.810417175292969 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 8.81087875366211 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 8.794075012207031 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 8.80522632598877 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 8.805710792541504 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 8.821784973144531 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 8.81722354888916 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 8.816720962524414 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 8.81328296661377 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 8.793083190917969 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 8.80716609954834 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 8.808812141418457 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 8.813474655151367 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 8.815420150756836 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 8.81128215789795 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 8.803872108459473 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 8.788902282714844 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 8.795353889465332 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 8.809257507324219 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 8.812219619750977 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 8.817944526672363 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 8.804923057556152 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 8.814324378967285 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 8.79733657836914 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 8.797942161560059 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 8.800788879394531 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 8.81674575805664 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 8.816137313842773 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 8.815010070800781 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 8.807985305786133 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 8.797608375549316 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 8.805769920349121 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 8.797648429870605 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 8.816141128540039 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 8.816350936889648 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 8.814350128173828 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 8.814248085021973 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 8.801103591918945 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 8.809121131896973 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 8.796737670898438 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 8.816267013549805 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 8.816548347473145 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 8.817562103271484 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 8.819507598876953 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 8.8024263381958 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 8.802618980407715 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 8.809596061706543 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 8.809789657592773 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 8.81056022644043 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 8.820535659790039 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 8.815603256225586 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 8.808366775512695 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 8.809601783752441 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 8.807040214538574 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 8.808938026428223 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 8.816487312316895 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 8.820469856262207 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 8.81482982635498 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 8.802994728088379 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 8.80931282043457 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 8.804902076721191 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 8.816726684570312 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 8.818363189697266 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 8.811543464660645 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 8.818028450012207 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 8.808598518371582 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 8.806276321411133 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 8.809063911437988 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 8.810417175292969 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 8.814981460571289 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 8.815838813781738 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 8.805549621582031 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 8.811816215515137 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 8.81287670135498 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 8.811849594116211 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 8.812233924865723 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 8.811664581298828 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 8.81186294555664 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 8.811786651611328 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 8.81282901763916 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 8.809282302856445 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 8.803751945495605 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 8.814863204956055 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 8.815808296203613 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 8.811635971069336 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 8.809222221374512 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 8.802786827087402 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 8.805046081542969 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 8.809950828552246 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 8.813745498657227 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 8.811663627624512 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 8.810026168823242 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 8.814306259155273 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 8.800541877746582 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 8.809263229370117 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 8.806507110595703 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 8.814821243286133 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 8.814559936523438 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 8.810361862182617 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 8.806573867797852 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 8.810230255126953 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 8.801827430725098 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 8.80456256866455 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 8.80566692352295 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 8.807615280151367 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 8.818575859069824 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 8.799636840820312 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 8.809732437133789 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 8.806486129760742 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 8.807699203491211 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 8.808717727661133 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 8.811003684997559 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 8.803584098815918 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 8.78859806060791 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 8.806620597839355 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 8.8106689453125 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 8.80951976776123 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 8.813204765319824 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 8.812081336975098 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 8.81395435333252 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 8.80932903289795 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 8.807367324829102 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 8.803083419799805 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 8.80740737915039 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 8.815792083740234 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 8.814421653747559 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 8.819856643676758 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 8.811405181884766 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 8.80759048461914 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 8.806985855102539 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 8.805380821228027 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 8.811824798583984 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 8.8093900680542 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 8.810291290283203 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 8.805983543395996 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 8.810190200805664 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 8.8086576461792 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 8.810752868652344 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 8.809921264648438 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 8.810169219970703 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 8.806859970092773 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 8.809663772583008 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 8.805736541748047 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 8.807938575744629 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 8.80529499053955 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 8.813436508178711 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 8.81142520904541 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 8.810759544372559 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 8.80864429473877 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 8.806456565856934 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 8.808202743530273 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 8.805948257446289 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 8.811173439025879 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 8.813843727111816 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 8.810041427612305 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 8.805280685424805 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 8.798263549804688 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 8.804847717285156 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 8.809725761413574 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 8.813831329345703 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 8.81237506866455 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 8.808098793029785 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 8.806746482849121 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 8.802026748657227 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 8.811274528503418 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 8.803166389465332 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 8.801030158996582 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 8.80607795715332 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 8.811463356018066 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 8.820696830749512 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 8.795037269592285 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 8.795187950134277 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 8.797684669494629 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 8.818655014038086 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 8.816838264465332 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 8.818058013916016 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 8.818338394165039 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 8.798913955688477 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 8.802285194396973 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 8.798582077026367 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 8.812288284301758 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 8.813514709472656 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 8.808375358581543 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 8.811001777648926 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 8.800189018249512 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 8.804289817810059 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 8.796107292175293 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 8.811381340026855 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 8.813936233520508 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 8.809279441833496 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 8.812973976135254 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 8.798349380493164 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 8.804119110107422 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 8.796282768249512 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 8.809866905212402 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 8.808755874633789 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 8.809927940368652 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 8.812235832214355 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 8.795612335205078 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 8.80369758605957 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 8.795951843261719 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 8.810803413391113 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 8.81446361541748 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 8.808938026428223 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 8.811923027038574 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 8.798324584960938 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 8.802681922912598 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 8.798171043395996 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 8.81237506866455 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 8.813408851623535 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 8.81179141998291 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 8.816242218017578 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 8.799880981445312 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 8.799676895141602 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 8.794901847839355 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 8.811749458312988 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 8.814295768737793 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 8.814444541931152 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 8.816719055175781 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 8.796086311340332 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 8.80302619934082 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 8.798917770385742 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 8.812406539916992 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 8.813597679138184 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 8.81130313873291 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 8.816119194030762 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 8.798519134521484 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 8.79788589477539 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 8.800466537475586 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 8.811718940734863 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 8.814770698547363 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 8.811585426330566 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 8.81667423248291 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 8.798208236694336 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 8.819605827331543 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 8.799829483032227 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 8.814136505126953 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 8.814249038696289 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 8.810710906982422 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 8.822196960449219 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 8.800701141357422 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 8.798104286193848 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 8.79921817779541 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 8.812481880187988 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 8.81380844116211 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 8.808877944946289 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 8.812135696411133 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 8.791539192199707 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 8.782092094421387 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 8.794260025024414 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 8.812947273254395 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 8.81490707397461 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 8.81627082824707 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 8.81506061553955 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 8.758246421813965 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 8.755531311035156 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 8.736128807067871 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.754146575927734 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.7553815841674805 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.757915496826172 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.763261318206787 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.757993221282959 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.758328914642334 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.7572150230407715 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.759875297546387 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.759774208068848 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.759853839874268 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.760338306427002 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.76072883605957 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.762120723724365 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.75960636138916 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.759790420532227 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.760222911834717 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.762721538543701 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.760251522064209 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.7616472244262695 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.7605743408203125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.760128974914551 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.761024475097656 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.761685371398926 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.758698463439941 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.759346008300781 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.757373332977295 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.759592056274414 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.75880765914917 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.76059103012085 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.759955883026123 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.7613959312438965 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.760817050933838 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.754817485809326 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.75698709487915 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.757224082946777 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.761105060577393 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.760930061340332 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.759223461151123 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.7595343589782715 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.7560954093933105 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.757216453552246 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.75771427154541 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.759429931640625 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.759979724884033 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.760380268096924 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.759242534637451 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.7552995681762695 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.757839202880859 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.758546829223633 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.7596211433410645 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.76078987121582 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.761610507965088 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.759200572967529 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.75523042678833 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.760364055633545 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.760787487030029 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.761626720428467 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.761550426483154 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.760138511657715 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.759174823760986 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.759091377258301 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.759976863861084 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.757175445556641 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.759566307067871 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.759898662567139 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.759711742401123 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.758194446563721 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.758665561676025 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.7564616203308105 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.757076263427734 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.760365962982178 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.759446144104004 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.760347366333008 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.758987903594971 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.757500648498535 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.75918436050415 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.7574076652526855 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.759923458099365 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.760523796081543 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.75949239730835 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.759655475616455 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.757826328277588 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.759879112243652 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.757558345794678 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.760819911956787 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.760476112365723 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.7617669105529785 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.759679317474365 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.755794525146484 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.759154796600342 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.759180068969727 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.760236740112305 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.760307312011719 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.759100914001465 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.759289741516113 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.7548604011535645 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.756832599639893 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.759614944458008 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.759958267211914 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.76138973236084 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.759655952453613 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.759397506713867 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.757187366485596 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.757471561431885 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.7569403648376465 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.759726524353027 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.7603960037231445 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.760634899139404 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.75856351852417 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.756349086761475 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.75823974609375 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.759298801422119 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.760236740112305 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.762084484100342 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.759467601776123 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.760103225708008 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.756654262542725 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.757593154907227 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.761732578277588 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.760341644287109 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.761265277862549 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.7602434158325195 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.759215354919434 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.757411003112793 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.760334491729736 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.759036540985107 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.758863925933838 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.7591729164123535 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.76014518737793 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.760970115661621 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.757200241088867 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.759402751922607 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.759315013885498 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.761265277862549 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.760711669921875 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.760560989379883 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.759362697601318 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.758675575256348 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.759207248687744 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.7580647468566895 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.759506702423096 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.7611188888549805 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.760751247406006 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.758745193481445 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.7592878341674805 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.758957386016846 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.759068965911865 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.761638641357422 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.759584426879883 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.76039981842041 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.758650779724121 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.760223865509033 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.759466648101807 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.761639595031738 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.762150287628174 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.760156154632568 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.760028839111328 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.75991153717041 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.758885383605957 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.758022785186768 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.7580180168151855 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.759049415588379 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.7598652839660645 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.759176731109619 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.7579169273376465 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.7572808265686035 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.758546829223633 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.757733345031738 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.759808540344238 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.760229110717773 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.759904384613037 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.759171009063721 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.756670951843262 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.758352756500244 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.759795665740967 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.760496616363525 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.760259628295898 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.7596940994262695 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.759120464324951 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.756954669952393 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.7588677406311035 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.758298397064209 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.758094310760498 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.7590718269348145 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.759073257446289 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.759533405303955 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.760008335113525 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.7568769454956055 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.760665416717529 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.759093284606934 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.759542942047119 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.758051872253418 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.758767127990723 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.75969934463501 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.758814334869385 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.759369850158691 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.760064601898193 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.758270263671875 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.760170936584473 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.758701801300049 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.757389068603516 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.755834102630615 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.759025573730469 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.760793209075928 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.75893497467041 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.7600274085998535 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.759400844573975 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.758969306945801 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.758243560791016 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.757940769195557 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.759310245513916 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.759580612182617 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.759385585784912 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.759713649749756 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.75913667678833 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.758747577667236 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.757800579071045 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.759045600891113 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.759303092956543 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.757750511169434 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.756492614746094 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.7569403648376465 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.759904861450195 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.758749008178711 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.758815765380859 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.760551452636719 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.758011817932129 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.758716106414795 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.75861930847168 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.756807804107666 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.758182525634766 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.760075569152832 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.760167598724365 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.758576393127441 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.758005142211914 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.755721569061279 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.7568440437316895 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.754517078399658 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.759552001953125 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.759336471557617 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.758139133453369 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.758856296539307 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.754586696624756 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.760705947875977 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.756326675415039 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.757787704467773 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.758161544799805 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.75870943069458 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.760092735290527 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.7540602684021 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.755928993225098 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.757438659667969 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.760416030883789 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.759723663330078 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.759029388427734 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.760060787200928 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.7534308433532715 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.7585320472717285 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.75625467300415 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.759462356567383 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.759812355041504 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.757610321044922 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.7590179443359375 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.756186485290527 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.758260726928711 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.758533954620361 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.758920192718506 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.759466648101807 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.757760524749756 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.761236190795898 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.754522323608398 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.755583763122559 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.753384590148926 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.759387969970703 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.758787631988525 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.758838176727295 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.759634017944336 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.754331111907959 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.756760120391846 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.75637149810791 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.759853839874268 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.760209560394287 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.760241508483887 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.760731220245361 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.75540828704834 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.756771564483643 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.7566375732421875 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.75941276550293 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.75958251953125 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.760010242462158 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.760667324066162 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.753894805908203 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.75627326965332 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.756508827209473 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.759793758392334 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.760148525238037 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.758204936981201 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.760838985443115 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.757038593292236 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.755207061767578 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.754809379577637 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.759109020233154 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.758251667022705 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.758399486541748 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.7613043785095215 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.754933834075928 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.755698204040527 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.755471229553223 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.758028507232666 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.756999969482422 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.757448196411133 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.7593231201171875 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.7577619552612305 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.756293773651123 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.754429340362549 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.760307788848877 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.7599921226501465 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.75587797164917 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.763233184814453 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.754937648773193 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.759326457977295 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.754900932312012 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.75900936126709 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.762235641479492 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.759779930114746 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.760296821594238 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.751812934875488 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.746415615081787 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.755228042602539 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.759606838226318 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.759973526000977 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.759479522705078 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.760740280151367 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.7430500984191895 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.74822998046875 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.738206386566162 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.264240264892578 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.263523101806641 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.266392230987549 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.268514633178711 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.265488147735596 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.265771865844727 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.265524864196777 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.266439914703369 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.266439914703369 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.26663064956665 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.266671657562256 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.265810966491699 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.266283988952637 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.266138076782227 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.266142845153809 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.266223907470703 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.266178607940674 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.266064167022705 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.265803813934326 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.266238689422607 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.266198635101318 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.266998767852783 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.266561031341553 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.266197204589844 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.265682697296143 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.265679359436035 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.266857624053955 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.265955448150635 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.266022682189941 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.266079902648926 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.265823841094971 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.266007900238037 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.264622211456299 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.264361381530762 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.265757083892822 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.266825199127197 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.266439437866211 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.266227722167969 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.265462875366211 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.263975143432617 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.265599250793457 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.264646053314209 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.266236782073975 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.266358852386475 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.265979766845703 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.266465187072754 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.264301776885986 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.265125274658203 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.264936447143555 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.265892505645752 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.265721321105957 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.265635013580322 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.265528678894043 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.2647809982299805 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.266740322113037 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.266458511352539 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.267142295837402 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.266568183898926 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.266824722290039 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.266510486602783 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.263962745666504 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.266467094421387 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.265570640563965 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.266469955444336 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.266477108001709 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.266792297363281 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.266443252563477 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.264711856842041 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.266256332397461 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.265313148498535 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.265564441680908 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.265714168548584 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.265387535095215 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.265385150909424 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.26563835144043 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.2653093338012695 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.265925884246826 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.265472888946533 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.2661261558532715 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.26564884185791 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.26582145690918 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.265768051147461 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.266818046569824 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.2657647132873535 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.266584873199463 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.266792297363281 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.265945911407471 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.266010284423828 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.264572620391846 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.2659687995910645 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.265841007232666 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.2658796310424805 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.266348361968994 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.266279220581055 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.266268253326416 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.265397071838379 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.265402317047119 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.265842437744141 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.266765117645264 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.266434192657471 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.266546249389648 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.26589298248291 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.265012264251709 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.265601634979248 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.263935089111328 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.266448497772217 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.266355514526367 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.266498565673828 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.265751361846924 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.263494491577148 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.265480995178223 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.266278266906738 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.266237735748291 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.2667317390441895 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.266757011413574 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.266726493835449 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.265466213226318 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.266159534454346 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.265837669372559 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.266899108886719 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.2667717933654785 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.266210079193115 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.266397953033447 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.265690803527832 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.266600608825684 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.265115737915039 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.26607084274292 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.2653350830078125 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.266146183013916 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.265117645263672 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.265612602233887 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.265013694763184 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.2660298347473145 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.265906810760498 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.266070365905762 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.266294956207275 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.2655534744262695 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.265567302703857 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.264162540435791 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.2658514976501465 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.265896320343018 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.266329765319824 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.266360282897949 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.265868186950684 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.265471935272217 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.266674041748047 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.265981197357178 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.2662577629089355 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.265658855438232 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.266605854034424 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.265554428100586 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.265200138092041 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.26622200012207 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.264717102050781 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.266169548034668 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.266067981719971 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.266236782073975 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.267543792724609 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.266911029815674 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.2659783363342285 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.265282154083252 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.266568183898926 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.266000747680664 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.266717910766602 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.2663164138793945 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.265279293060303 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.265827655792236 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.265693664550781 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.266421794891357 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.266701698303223 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.26597261428833 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.2665324211120605 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.265933990478516 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.265889644622803 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.264856815338135 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.266420841217041 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.26628303527832 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.266980171203613 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.26641321182251 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.266049385070801 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.264626502990723 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.26560640335083 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.266485691070557 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.266140460968018 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.266646862030029 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.266186714172363 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.265732765197754 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.266019821166992 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.264994144439697 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.266900539398193 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.267142295837402 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.265841007232666 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.266485691070557 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.266206741333008 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.266164302825928 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.265976905822754 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.266814708709717 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.266031742095947 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.266778945922852 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.265456676483154 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.266283988952637 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.266392230987549 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.266790390014648 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.266778945922852 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.2659783363342285 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.266114234924316 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.265574932098389 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.266468524932861 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.266340255737305 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.267293453216553 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.26604700088501 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.266096115112305 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.265822887420654 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.265663146972656 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.265515327453613 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.265847206115723 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.265750408172607 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.266725063323975 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.265722751617432 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.266031742095947 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.266386032104492 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.264995574951172 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.265627861022949 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.265336036682129 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.266364574432373 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.265872955322266 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.2667059898376465 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.265923023223877 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.265456199645996 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.266245365142822 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.264681816101074 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.266096115112305 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.2664594650268555 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.265913009643555 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.2659807205200195 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.264932632446289 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.2650580406188965 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.2652482986450195 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.266186237335205 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.266049861907959 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.2658915519714355 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.266180038452148 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.263467311859131 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.266244411468506 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.263047218322754 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.266082286834717 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.265479564666748 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.265110015869141 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.265979290008545 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.262718677520752 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.265852928161621 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.262762069702148 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.266469955444336 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.267085552215576 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.266345024108887 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.26610803604126 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.26457405090332 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.264398097991943 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.264797687530518 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.2660064697265625 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.266506195068359 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.264867782592773 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.266157627105713 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.262924671173096 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.264899730682373 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.26359224319458 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.26627254486084 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.265944004058838 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.26583194732666 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.266397953033447 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.264671802520752 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.266471862792969 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.264056205749512 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.265552520751953 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.266215801239014 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.266200542449951 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.266051769256592 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.26317024230957 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.264214515686035 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.263428688049316 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.266094207763672 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.266607284545898 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.264901161193848 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.265911102294922 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.264826774597168 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.264438629150391 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.264128684997559 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.266299724578857 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.266510486602783 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.265468120574951 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.267551898956299 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.263893127441406 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.26392126083374 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.263154983520508 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.266593933105469 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.2665581703186035 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.266422748565674 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.267052173614502 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.262949466705322 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.264638423919678 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.264498233795166 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.2659993171691895 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.266247272491455 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.264739990234375 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.266112327575684 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.264332294464111 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.26447868347168 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.264511585235596 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.266147613525391 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.264920711517334 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.2657880783081055 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.266230583190918 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.26418924331665 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.264086723327637 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.264894008636475 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.266438007354736 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.26661491394043 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.2655816078186035 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.26666784286499 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.2631449699401855 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.263803482055664 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.264935493469238 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.269292831420898 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.266831398010254 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.265848159790039 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.267354488372803 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.263106346130371 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.2640204429626465 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.26502799987793 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.266085624694824 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.26599645614624 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.267209529876709 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.2669358253479 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.256406307220459 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.258304595947266 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.252650260925293 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.037781238555908 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.03796911239624 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.039759159088135 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.039947986602783 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.039631366729736 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.039458274841309 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.039304733276367 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.039770603179932 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.03990364074707 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.039984226226807 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.039978504180908 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.039669513702393 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.0401129722595215 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.040040016174316 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.039741516113281 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.039919853210449 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.039971351623535 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.040064811706543 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.039346218109131 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.039968490600586 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.039988040924072 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.040037155151367 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.039857864379883 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.0395636558532715 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.03963565826416 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.039438247680664 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.0398664474487305 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.039381980895996 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.039606094360352 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.039698600769043 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.0397725105285645 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.0395989418029785 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.039237022399902 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.038244724273682 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.039134502410889 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.039653301239014 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.039954662322998 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.040110111236572 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.039369106292725 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.039488792419434 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.0392656326293945 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.039618492126465 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.03973913192749 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.039727687835693 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.039330959320068 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.039396286010742 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.038371562957764 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.039475917816162 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.039811134338379 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.039605617523193 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.039724826812744 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.0395965576171875 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.0391387939453125 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.039245128631592 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.038836479187012 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.0401716232299805 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.040173053741455 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.04002046585083 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.039887428283691 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.0400590896606445 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.039251327514648 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.039605617523193 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.04002571105957 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.040309906005859 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.040129661560059 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.039864540100098 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.039881229400635 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.039527893066406 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.039710521697998 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.039963722229004 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.03963565826416 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.0395941734313965 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.039741039276123 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.039540767669678 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.039438724517822 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.039318561553955 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.040571689605713 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.039885997772217 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.039660930633545 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.03969669342041 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.039366722106934 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.039100170135498 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.040599346160889 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.039480209350586 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.040248870849609 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.039944171905518 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.040245056152344 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.039542198181152 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.039059638977051 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.039189338684082 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.039556503295898 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.039906978607178 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.04019832611084 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.039707660675049 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.039304733276367 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.038139820098877 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.039747714996338 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.039350509643555 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.039815902709961 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.039892196655273 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.039444446563721 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.040121078491211 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.038656711578369 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.039492130279541 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.040318489074707 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.039910316467285 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.039987087249756 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.039775371551514 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.039822578430176 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.039017200469971 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.039958477020264 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.037341594696045 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.040327548980713 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.0403265953063965 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.039492130279541 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.039347171783447 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.038830757141113 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.039614200592041 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.038674354553223 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.039872646331787 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.039926052093506 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.0399909019470215 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.039711952209473 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.037968635559082 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.039534091949463 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.039907932281494 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.03963565826416 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.039825439453125 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.039648532867432 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.040136814117432 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.038780212402344 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.03974723815918 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.039069652557373 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.039755344390869 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.040132522583008 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.040231704711914 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.0405426025390625 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.03975248336792 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.039642810821533 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.038541316986084 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.039750576019287 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.039926528930664 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.039638519287109 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.0400519371032715 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.039913654327393 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.039573669433594 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.039162635803223 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.039798259735107 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.039945602416992 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.039642810821533 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.039813995361328 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.038740634918213 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.038233280181885 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.040240287780762 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.039809703826904 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.039747714996338 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.040030479431152 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.03942346572876 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.038995742797852 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.0383405685424805 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.039794445037842 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.040174961090088 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.039762496948242 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.040043354034424 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.039941310882568 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.0397796630859375 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.039905071258545 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.039239883422852 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.040112018585205 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.039972305297852 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.039827346801758 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.039107322692871 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.039559364318848 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.038356781005859 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.040129661560059 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.040107250213623 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.039981365203857 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.039649963378906 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.039524555206299 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.039511680603027 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.038960933685303 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.0389533042907715 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.039963722229004 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.039239883422852 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.040329933166504 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.039444446563721 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.0387797355651855 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.039644718170166 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.039487361907959 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.039675235748291 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.03985595703125 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.0397796630859375 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.039863109588623 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.040126800537109 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.039444446563721 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.038936138153076 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.039618492126465 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.039412498474121 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.039778232574463 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.04007625579834 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.03955078125 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.0395379066467285 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.037922382354736 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.040031909942627 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.039443016052246 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.040139198303223 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.040395259857178 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.039275646209717 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.039610862731934 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.039280414581299 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.039972305297852 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.039450645446777 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.040297031402588 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.039527893066406 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.03884220123291 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.038915157318115 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.0392937660217285 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.03951358795166 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.039300441741943 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.0393476486206055 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.039351940155029 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.0385212898254395 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.039945602416992 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.039187908172607 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.039874076843262 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.0401482582092285 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.0395050048828125 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.0395379066467285 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.03834867477417 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.039706707000732 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.039409637451172 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.0399885177612305 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.040116786956787 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.039944171905518 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.039802551269531 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.03897762298584 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.038829326629639 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.038825511932373 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.040217399597168 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.039718151092529 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.039004802703857 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.039706707000732 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.038801193237305 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.039228439331055 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.039556503295898 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.039283752441406 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.039879322052002 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.039413928985596 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.040095329284668 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.038506984710693 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.038696765899658 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.038937568664551 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.040042400360107 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.040086269378662 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.0397725105285645 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.039551258087158 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.03796911239624 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.038563251495361 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.039140701293945 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.0394062995910645 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.040014266967773 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.03965950012207 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.039795875549316 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.038843154907227 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.039493560791016 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.0384745597839355 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.039747714996338 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.039799213409424 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.039230823516846 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.040623664855957 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.038239002227783 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.038895606994629 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.038861274719238 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.039681911468506 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.039139747619629 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.0390543937683105 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.039872646331787 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.038041591644287 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.039412498474121 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.038639545440674 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.040038585662842 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.0400495529174805 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.039841651916504 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.040041923522949 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.038372039794922 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.038872718811035 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.037075519561768 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.039968490600586 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.039539813995361 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.039238452911377 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.039344310760498 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.038855075836182 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.03922700881958 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.0385212898254395 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.040112018585205 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.039815902709961 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.0399250984191895 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.0405659675598145 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.037843704223633 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.038236618041992 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.038201332092285 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.040340900421143 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.039360046386719 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.0394415855407715 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.0407514572143555 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.038449287414551 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.037461757659912 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.038767337799072 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.039679050445557 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.039728164672852 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.039241313934326 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.040011405944824 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.038679599761963 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.039231777191162 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.038347244262695 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.039971351623535 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.040416240692139 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.038783073425293 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.040424346923828 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.038431644439697 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.038625717163086 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.038376331329346 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.040506839752197 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.040518283843994 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.040030479431152 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.040442943572998 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.037083148956299 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.036137580871582 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.036500930786133 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.039935111999512 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.040157794952393 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.039644718170166 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.039935111999512 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.035271167755127 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.0347771644592285 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.035180568695068 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.889880180358887 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.889847755432129 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.890834331512451 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.891274452209473 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.890127182006836 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.8904924392700195 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.889919757843018 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.890408039093018 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.890503406524658 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.8905110359191895 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.890622615814209 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.890327453613281 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.890475273132324 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.890470027923584 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.890466690063477 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.890522003173828 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.8907036781311035 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.890693187713623 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.890076637268066 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.890476226806641 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.890252590179443 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.890779972076416 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.890761852264404 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.890711307525635 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.890436172485352 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.890441417694092 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.890617370605469 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.890209674835205 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.890559196472168 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.890336513519287 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.890470027923584 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.890274524688721 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.8900146484375 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.890270233154297 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.8900370597839355 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.890721797943115 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.890803337097168 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.890273094177246 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.8906779289245605 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.889776706695557 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.890451431274414 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.889385223388672 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.890631198883057 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.890549182891846 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.890496730804443 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.890393257141113 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.889806747436523 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.8902387619018555 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.890012741088867 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.890291213989258 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.890216827392578 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.890377044677734 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.890329360961914 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.890214443206787 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.890822410583496 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.889392852783203 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.890761375427246 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.8906073570251465 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.890246868133545 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.890671730041504 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.890215873718262 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.890190601348877 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.889411926269531 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.890742301940918 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.890629768371582 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.890691757202148 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.890422821044922 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.889738082885742 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.89027214050293 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.8898420333862305 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.89021635055542 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.890135288238525 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.890584945678711 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.890049457550049 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.890532970428467 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.890664100646973 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.889997959136963 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.8903727531433105 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.890549659729004 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.890639305114746 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.890562534332275 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.890108585357666 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.889978408813477 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.890269756317139 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.890761852264404 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.890847682952881 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.890583515167236 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.890939235687256 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.889762878417969 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.890100479125977 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.890470504760742 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.890305519104004 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.890501976013184 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.890569686889648 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.890441417694092 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.889835357666016 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.890449047088623 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.890477657318115 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.890945911407471 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.890679359436035 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.89072847366333 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.890488624572754 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.8902435302734375 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.8906731605529785 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.889786243438721 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.890934944152832 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.890480041503906 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.890301704406738 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.890493869781494 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.890002250671387 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.890163898468018 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.891199588775635 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.890564918518066 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.8910231590271 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.890896320343018 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.89043664932251 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.890538692474365 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.89051628112793 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.89089822769165 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.890771389007568 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.890488624572754 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.890629768371582 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.890653610229492 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.890344619750977 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.890683174133301 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.890499114990234 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.890421390533447 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.89032506942749 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.890777587890625 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.889975547790527 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.8900227546691895 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.890621185302734 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.8908305168151855 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.890594005584717 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.890340328216553 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.890443325042725 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.8903632164001465 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.890814304351807 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.891132831573486 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.890284061431885 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.890535354614258 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.8903727531433105 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.890641212463379 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.889830112457275 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.890028953552246 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.890107154846191 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.890822410583496 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.890274524688721 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.890447616577148 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.890928268432617 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.889795780181885 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.890616416931152 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.890633583068848 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.890157699584961 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.890645980834961 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.890468597412109 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.890666961669922 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.889947891235352 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.89068603515625 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.891688346862793 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.889889240264893 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.890261173248291 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.890789985656738 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.890596389770508 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.890146255493164 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.890837669372559 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.890291213989258 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.889983654022217 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.890342712402344 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.890584468841553 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.890295028686523 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.890224456787109 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.890559196472168 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.891195297241211 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.889910697937012 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.890544891357422 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.890691757202148 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.8908586502075195 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.890426158905029 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.889767169952393 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.890138149261475 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.889810085296631 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.8902716636657715 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.8906450271606445 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.8904619216918945 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.890005588531494 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.890244960784912 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.888939380645752 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.890292644500732 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.890766143798828 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.8911919593811035 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.89095401763916 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.890644073486328 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.889995098114014 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.890825271606445 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.890783786773682 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.890555381774902 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.890524387359619 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.890412330627441 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.890326499938965 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.890211582183838 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.890650749206543 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.890701770782471 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.890755653381348 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.890395164489746 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.890769958496094 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.890544414520264 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.8903889656066895 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.890166759490967 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.8907599449157715 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.89052677154541 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.890571594238281 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.890625476837158 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.890601634979248 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.890442848205566 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.890788555145264 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.890687465667725 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.8905720710754395 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.890565395355225 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.890582084655762 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.89003849029541 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.890225410461426 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.890349388122559 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.889847755432129 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.890542984008789 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.890817642211914 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.890171527862549 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.890486240386963 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.890894889831543 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.89023494720459 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.8898420333862305 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.890489101409912 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.8903021812438965 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.890379905700684 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.8898115158081055 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.889985084533691 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.8903303146362305 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.8898210525512695 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.890504360198975 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.8903703689575195 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.890414714813232 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.890192031860352 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.890106678009033 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.89008092880249 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.889126300811768 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.890400409698486 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.8904008865356445 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.889753341674805 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.8909077644348145 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.889132976531982 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.890301704406738 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.889316082000732 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.890337944030762 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.890427589416504 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.890356063842773 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.890891075134277 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.889451503753662 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.890198707580566 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.8892693519592285 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.890408515930176 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.889914512634277 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.889863967895508 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.890493869781494 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.889180660247803 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.8896636962890625 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.889791488647461 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.890523910522461 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.8902177810668945 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.890236854553223 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.890544414520264 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.888978481292725 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.890189170837402 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.888585567474365 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.890503406524658 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.890232086181641 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.890590667724609 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.890497207641602 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.889394760131836 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.890246391296387 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.889803886413574 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.890665054321289 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.890963077545166 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.890214443206787 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.890671730041504 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.889274597167969 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.889820098876953 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.890097141265869 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.890312671661377 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.890621185302734 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.890468597412109 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.891176223754883 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.889364719390869 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.8894124031066895 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.889788627624512 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.890499114990234 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.890567302703857 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.890242099761963 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.890947341918945 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.889540195465088 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.8898115158081055 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.8891167640686035 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.890491962432861 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.890361785888672 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.89030647277832 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.890615940093994 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.889601230621338 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.8906426429748535 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.889120101928711 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.890343189239502 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.8903303146362305 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.890075206756592 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.890449523925781 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.888828754425049 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.889781951904297 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.889583587646484 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.890369892120361 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.890392303466797 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.890384674072266 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.891379356384277 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.889107704162598 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.889533042907715 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.890075206756592 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.891170978546143 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.890471935272217 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.890500545501709 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.890851974487305 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.889235496520996 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.8894195556640625 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.889946937561035 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.890474796295166 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.890399932861328 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.8910698890686035 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.890641689300537 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.887609481811523 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.886228561401367 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.8855881690979 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.798614025115967 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.798622131347656 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.799726963043213 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.799978733062744 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.79962158203125 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.799586772918701 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.7994303703308105 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.799557685852051 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.799706935882568 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.79964017868042 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.799539566040039 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.799686908721924 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.799955368041992 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.799531936645508 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.799648284912109 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.79959774017334 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.799736976623535 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.799330234527588 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.799582481384277 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.7997307777404785 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.799595832824707 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.79945707321167 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.799732208251953 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.799595832824707 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.799692630767822 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.799415111541748 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.799986839294434 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.799835681915283 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.79941987991333 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.799454689025879 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.799766540527344 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.799689769744873 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.799354076385498 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.799140453338623 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.799368858337402 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.799540042877197 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.799659252166748 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.799500465393066 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.799157619476318 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.799098014831543 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.799779415130615 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.799811363220215 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.799421310424805 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.799392223358154 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.799206256866455 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.799994468688965 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.799349308013916 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.799436569213867 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.799524307250977 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.799524307250977 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.799470901489258 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.799325942993164 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.7997870445251465 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.799068927764893 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.799231052398682 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.799678802490234 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.799711227416992 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.799646854400635 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.799778938293457 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.799449920654297 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.799435138702393 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.799708366394043 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.799579620361328 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.799718379974365 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.799718379974365 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.7996368408203125 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.799572467803955 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.799211502075195 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.799319267272949 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.799435615539551 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.799429416656494 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.79932165145874 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.79945707321167 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.799549102783203 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.799645900726318 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.799208641052246 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.800202369689941 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.799365043640137 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.7996039390563965 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.7996978759765625 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.799344539642334 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.7994256019592285 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.799808979034424 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.799644470214844 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.79959774017334 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.799654483795166 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.799767017364502 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.799536228179932 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.7992119789123535 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.799804210662842 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.799464702606201 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.799680233001709 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.799715995788574 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.799296855926514 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.798919677734375 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.79915714263916 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.799041748046875 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.799219608306885 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.799756050109863 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.799663543701172 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.799350261688232 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.799496173858643 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.79839563369751 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.798754692077637 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.799325466156006 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.799539566040039 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.799783706665039 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.799366474151611 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.799461841583252 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.798709392547607 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.799007892608643 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.799126148223877 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.79971170425415 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.7997660636901855 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.799596309661865 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.7995805740356445 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.798974990844727 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.799371719360352 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.7986955642700195 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.799777030944824 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.79987907409668 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.799566268920898 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.799643039703369 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.799154281616211 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.7992167472839355 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.79905366897583 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.799546241760254 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.799590110778809 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.798901557922363 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.799659252166748 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.799135684967041 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.799756050109863 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.798847675323486 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.799344062805176 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.79975700378418 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.799440383911133 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.7994465827941895 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.799349308013916 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.79869270324707 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.799308776855469 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.7994184494018555 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.799543380737305 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.79947566986084 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.799673080444336 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.799508571624756 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.799140453338623 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.798757553100586 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.799738883972168 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.799494743347168 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.799737453460693 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.799349308013916 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.798991680145264 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.798620700836182 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.799884796142578 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.799341201782227 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.7993011474609375 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.799738883972168 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.799577236175537 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.7989888191223145 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.798672199249268 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.799500465393066 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.800049304962158 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.799648284912109 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.799561500549316 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.799678802490234 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.798734188079834 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.799410343170166 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.799293518066406 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.799830913543701 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.7995429039001465 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.799571514129639 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.799678802490234 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.799729824066162 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.798924922943115 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.79907751083374 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.799642086029053 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.799586772918701 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.799529075622559 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.799657344818115 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.799652576446533 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.798938751220703 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.799136638641357 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.799756050109863 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.799442768096924 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.799442768096924 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.79966926574707 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.79918098449707 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.799960613250732 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.798698425292969 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.799352645874023 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.799655437469482 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.799594879150391 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.799807071685791 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.799274921417236 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.799264907836914 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.798604965209961 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.799685001373291 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.799695014953613 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.799565315246582 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.799393653869629 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.799914836883545 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.799299716949463 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.799204349517822 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.799619197845459 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.799427032470703 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.7997589111328125 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.799332618713379 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.799628734588623 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.799779415130615 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.798985958099365 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.799703598022461 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.799185276031494 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.799345970153809 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.799680709838867 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.798934459686279 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.7995500564575195 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.798902988433838 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.799430847167969 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.79926872253418 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.79927921295166 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.799631595611572 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.799304008483887 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.799292087554932 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.799546718597412 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.799708843231201 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.799865245819092 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.799865245819092 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.799294948577881 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.798956871032715 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.799288749694824 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.798973083496094 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.799447059631348 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.799736976623535 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.799670219421387 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.799515247344971 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.798706531524658 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.799304008483887 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.798819541931152 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.799894332885742 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.799646854400635 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.799185752868652 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.799895763397217 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.79895544052124 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.7996039390563965 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.799903869628906 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.799648284912109 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.799506664276123 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.799450874328613 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.7993364334106445 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.798924922943115 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.799496173858643 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.799030780792236 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.799772262573242 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.799657344818115 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.799678802490234 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.799466609954834 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.799801826477051 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.799176216125488 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.79995584487915 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.799612045288086 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.799849510192871 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.7995076179504395 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.799671649932861 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.798833847045898 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.799376010894775 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.799013614654541 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.799566745758057 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.799713611602783 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.799646854400635 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.7998366355896 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.7989935874938965 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.799041271209717 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.799371719360352 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.799323081970215 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.799565315246582 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.799494743347168 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.799427032470703 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.798614025115967 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.7992730140686035 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.798109531402588 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.799746036529541 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.799558639526367 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.7996134757995605 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.79963493347168 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.798952579498291 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.799283504486084 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.798844814300537 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.799911975860596 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.799511432647705 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.799677848815918 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.799561500549316 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.798876762390137 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.799290180206299 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.799057960510254 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.7996649742126465 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.799790382385254 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.799637794494629 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.799663066864014 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.798680782318115 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.7987895011901855 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.79948091506958 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.7997355461120605 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.79967737197876 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.799345970153809 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.799715995788574 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.798757553100586 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.798833847045898 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.799107551574707 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.799686908721924 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.799454212188721 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.7994608879089355 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.799883842468262 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.799431800842285 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.799150466918945 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.798558712005615 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.799761772155762 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.799938201904297 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.799399375915527 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.799629211425781 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.798875331878662 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.798856258392334 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.7989983558654785 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.799725532531738 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.800166130065918 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.799502372741699 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.799731731414795 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.798575401306152 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.7982988357543945 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.7985920906066895 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.799703598022461 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.799963474273682 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.799472808837891 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.79973840713501 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.797502517700195 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.797510623931885 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.797173500061035 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.7287492752075195 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.729007720947266 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.729936599731445 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.7303643226623535 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.729442119598389 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.729540824890137 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.7291998863220215 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.729360580444336 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.729432582855225 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.729462146759033 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.729524612426758 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.729516506195068 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.729471206665039 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.729564189910889 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.729559421539307 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.729475975036621 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.7295732498168945 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.729687213897705 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.7294487953186035 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.729549884796143 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.729590892791748 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.729652404785156 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.729529857635498 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.729607105255127 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.729198455810547 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.7292985916137695 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.729626178741455 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.729248046875 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.7295308113098145 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.7293314933776855 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.729230880737305 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.72944974899292 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.729253768920898 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.729262351989746 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.729429721832275 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.729626178741455 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.729404926300049 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.72938346862793 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.729562282562256 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.729276657104492 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.729630470275879 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.729183197021484 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.729485511779785 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.72961950302124 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.729671955108643 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.729260444641113 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.729524612426758 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.72887659072876 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.729271411895752 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.7294816970825195 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.7292094230651855 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.729576110839844 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.729269981384277 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.729167938232422 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.729623794555664 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.729195594787598 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.729510307312012 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.729442596435547 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.729709625244141 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.729342937469482 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.729696750640869 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.729311466217041 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.7291107177734375 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.72946834564209 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.729526996612549 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.729556560516357 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.729533672332764 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.728811264038086 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.7294206619262695 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.729238510131836 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.729086875915527 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.72935152053833 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.729803562164307 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.728991508483887 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.729290962219238 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.729309558868408 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.728514671325684 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.729323387145996 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.729273796081543 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.729562282562256 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.729551315307617 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.729104518890381 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.729405403137207 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.730020523071289 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.7295331954956055 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.729652404785156 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.729386329650879 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.729576110839844 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.728899002075195 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.7292633056640625 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.729575157165527 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.7295379638671875 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.729456424713135 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.729485511779785 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.729677677154541 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.729369640350342 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.729567527770996 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.729290962219238 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.7295379638671875 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.729532241821289 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.7295050621032715 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.729533672332764 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.729913234710693 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.73007869720459 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.729374408721924 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.729686260223389 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.729635238647461 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.729574680328369 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.7290472984313965 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.729868412017822 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.729696750640869 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.729613780975342 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.729705810546875 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.729587078094482 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.729637622833252 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.729397296905518 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.728986740112305 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.729703426361084 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.729218006134033 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.729338645935059 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.729535102844238 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.7295427322387695 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.729310989379883 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.729475975036621 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.729559421539307 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.7295732498168945 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.7292585372924805 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.729341983795166 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.729687690734863 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.729453086853027 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.729340553283691 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.729029655456543 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.729371547698975 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.7294230461120605 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.729304790496826 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.7297821044921875 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.729477405548096 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.729409217834473 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.729523658752441 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.729040622711182 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.729462623596191 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.729585647583008 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.729386329650879 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.729103088378906 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.728768348693848 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.729376316070557 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.729936599731445 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.729386329650879 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.729315280914307 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.729365348815918 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.729011535644531 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.729433059692383 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.729752063751221 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.729338645935059 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.7295308113098145 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.729369640350342 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.729315280914307 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.729180335998535 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.729589939117432 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.7294793128967285 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.728804111480713 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.7292399406433105 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.729405879974365 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.729682445526123 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.729729652404785 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.729857444763184 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.729431629180908 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.729156494140625 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.729480266571045 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.729486465454102 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.729674816131592 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.729362964630127 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.729101657867432 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.729757308959961 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.729189395904541 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.7294745445251465 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.729380130767822 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.729806423187256 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.729167938232422 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.728740692138672 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.7293477058410645 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.729532241821289 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.729579448699951 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.729491710662842 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.729608535766602 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.728817462921143 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.728837013244629 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.728595733642578 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.729338645935059 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.729588508605957 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.729887962341309 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.729434967041016 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.729518413543701 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.729636192321777 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.729409694671631 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.729641437530518 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.729499340057373 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.729156970977783 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.729929447174072 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.729742527008057 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.729268550872803 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.7291154861450195 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.729613304138184 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.729984283447266 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.729406356811523 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.729476451873779 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.7295637130737305 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.729556560516357 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.729302883148193 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.72962760925293 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.729611396789551 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.729647159576416 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.729655742645264 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.72938871383667 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.7294487953186035 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.729382038116455 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.729414463043213 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.729661464691162 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.729548454284668 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.729429721832275 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.729408264160156 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.729400634765625 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.729344844818115 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.729149341583252 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.729473114013672 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.729807376861572 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.729379177093506 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.729672908782959 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.729510307312012 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.729868412017822 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.729247570037842 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.72957181930542 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.729272842407227 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.729214191436768 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.729667663574219 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.72934627532959 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.729496479034424 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.729098320007324 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.729536056518555 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.729581356048584 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.729444980621338 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.729308128356934 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.728716850280762 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.729662895202637 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.728760242462158 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.729613304138184 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.729535102844238 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.729088306427002 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.729585647583008 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.7286458015441895 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.729167461395264 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.728719711303711 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.729355812072754 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.729475975036621 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.729366302490234 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.729769706726074 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.728963851928711 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.728761196136475 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.728594779968262 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.729567527770996 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.729645729064941 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.729061603546143 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.729437828063965 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.728684902191162 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.729203701019287 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.729067325592041 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.72977352142334 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.7295145988464355 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.7291975021362305 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.729814052581787 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.728910446166992 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.72921085357666 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.728773593902588 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.729525566101074 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.7295308113098145 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.729775905609131 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.729581356048584 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.728658199310303 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.729099750518799 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.729350566864014 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.729835510253906 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.729599475860596 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.729404926300049 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.729696273803711 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.729055404663086 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.72901725769043 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.729249477386475 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.729349613189697 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.72968864440918 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.729358673095703 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.729957580566406 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.7291154861450195 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.72929048538208 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.729121208190918 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.729616641998291 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.729536056518555 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.7294697761535645 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.729726314544678 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.728882312774658 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.729403972625732 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.72891902923584 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.729496002197266 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.7296223640441895 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.72961950302124 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.729722023010254 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.729459762573242 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.729029655456543 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.729206562042236 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.729461193084717 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.729434013366699 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.729568958282471 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.729437828063965 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.728757381439209 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.729297161102295 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.729476451873779 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.729604721069336 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.729585647583008 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.729949951171875 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.729589939117432 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.729021072387695 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.7291951179504395 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.729063987731934 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.729651927947998 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.729699611663818 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.729612827301025 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.729659557342529 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.729135036468506 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.729603290557861 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.729255199432373 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.729581356048584 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.729718208312988 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.729917526245117 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.729696273803711 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.7274861335754395 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.727830410003662 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.72840690612793 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.679567813873291 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.6793341636657715 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.680228233337402 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.679928779602051 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.680284023284912 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.680100440979004 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.680121421813965 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.680132865905762 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.680266857147217 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.680000305175781 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.680018424987793 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.679980754852295 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.679941177368164 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.6800618171691895 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.680008888244629 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.680020332336426 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.680093765258789 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.6800537109375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.679924011230469 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.6802215576171875 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.680274486541748 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.680037975311279 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.680120468139648 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.680090427398682 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.680370807647705 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.680078029632568 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.6802287101745605 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.6802287101745605 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.679983615875244 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.680133819580078 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.6802473068237305 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.680370807647705 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.679884433746338 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.679786205291748 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.679836750030518 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.680119514465332 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.680046558380127 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.679922580718994 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.680073261260986 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.680098533630371 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.680348873138428 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.6800360679626465 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.6799750328063965 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.680018424987793 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.679875373840332 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.680204391479492 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.679907321929932 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.680499076843262 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.679950714111328 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.680022239685059 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.680098533630371 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.679890155792236 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.679922580718994 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.680206775665283 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.6800456047058105 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.679816722869873 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.680150985717773 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.679928779602051 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.679858207702637 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.680077075958252 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.679774761199951 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.680144309997559 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.68025541305542 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.68010950088501 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.68018102645874 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.680140972137451 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.68019962310791 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.679966926574707 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.680158615112305 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.679858684539795 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.680118560791016 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.680038928985596 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.679945468902588 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.680022239685059 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.67940092086792 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.68037223815918 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.680603504180908 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.680044174194336 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.6801581382751465 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.68002462387085 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.679986953735352 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.679835796356201 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.680515766143799 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.679862976074219 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.680110454559326 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.680078983306885 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.680306911468506 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.679895401000977 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.680129528045654 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.6797776222229 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.680314064025879 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.680262565612793 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.680148601531982 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.680297374725342 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.679820537567139 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.679793834686279 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.679930210113525 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.680323123931885 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.680149555206299 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.6800618171691895 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.680120468139648 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.679994106292725 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.679502487182617 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.6789422035217285 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.679854393005371 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.67995023727417 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.679849147796631 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.680231094360352 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.680516719818115 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.679515838623047 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.679643154144287 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.679745197296143 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.680060386657715 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.6803083419799805 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.679763317108154 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.680491924285889 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.680037498474121 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.679423809051514 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.680139541625977 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.680237770080566 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.680156707763672 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.680236339569092 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.6800360679626465 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.67953634262085 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.679732799530029 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.680224418640137 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.680366039276123 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.680105209350586 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.679727077484131 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.680170059204102 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.679977893829346 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.680421829223633 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.680072784423828 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.680089473724365 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.680196285247803 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.6799116134643555 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.679986476898193 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.680289268493652 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.679966926574707 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.680318832397461 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.6801676750183105 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.679934501647949 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.680123329162598 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.680078983306885 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.680634021759033 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.6800127029418945 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.679568290710449 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.680019378662109 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.6800079345703125 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.6802239418029785 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.680084228515625 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.679645538330078 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.680016040802002 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.6802263259887695 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.680050849914551 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.67995548248291 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.6804518699646 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.680038928985596 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.679616451263428 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.679714679718018 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.680228233337402 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.680355548858643 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.680273056030273 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.680274486541748 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.679989814758301 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.679644584655762 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.679895877838135 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.68010139465332 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.6799845695495605 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.680019378662109 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.6802778244018555 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.6796040534973145 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.68019962310791 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.679345607757568 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.679980754852295 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.680100440979004 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.680079936981201 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.680392742156982 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.6801981925964355 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.680074214935303 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.679734230041504 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.679609298706055 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.680352687835693 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.6800007820129395 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.68026065826416 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.680397987365723 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.6796746253967285 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.68035364151001 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.679931640625 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.680231094360352 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.680367946624756 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.680309772491455 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.680465221405029 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.679731369018555 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.679877758026123 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.679865837097168 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.680125713348389 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.680361270904541 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.680150985717773 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.679908752441406 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.680046558380127 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.680495262145996 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.679307460784912 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.6802215576171875 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.679871559143066 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.680427551269531 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.679980754852295 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.679986476898193 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.6799468994140625 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.680129528045654 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.680093765258789 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.6799821853637695 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.680110454559326 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.6802825927734375 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.679782867431641 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.6799468994140625 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.680048942565918 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.679863452911377 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.679670333862305 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.680287837982178 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.67954683303833 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.679913520812988 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.680085182189941 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.679746627807617 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.68013858795166 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.680133819580078 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.680219650268555 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.679945945739746 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.6796345710754395 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.6798505783081055 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.68024206161499 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.680091381072998 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.680062294006348 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.680124282836914 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.680136203765869 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.679442405700684 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.67971134185791 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.679986000061035 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.680303573608398 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.67997932434082 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.680064678192139 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.679887771606445 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.679519176483154 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.679656982421875 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.679391384124756 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.680064678192139 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.6802473068237305 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.680091381072998 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.679994106292725 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.679832935333252 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.679638385772705 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.679646015167236 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.680145740509033 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.680109024047852 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.680065155029297 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.679873466491699 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.679452896118164 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.679925441741943 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.68018913269043 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.6798858642578125 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.680072784423828 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.680060386657715 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.680005073547363 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.6800079345703125 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.680049419403076 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.679887771606445 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.680057048797607 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.680140972137451 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.67999267578125 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.680011749267578 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.679492950439453 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.679718971252441 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.679682731628418 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.680130958557129 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.679881572723389 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.679754257202148 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.68001651763916 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.679760932922363 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.6796064376831055 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.679179668426514 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.680046558380127 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.680083274841309 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.6799726486206055 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.680013179779053 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.67921781539917 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.679581165313721 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.679492950439453 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.680317401885986 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.679919719696045 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.680206775665283 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.680226802825928 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.679762363433838 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.679590702056885 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.679580211639404 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.680150508880615 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.679849624633789 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.67993688583374 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.680142879486084 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.679546356201172 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.679399490356445 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.679893493652344 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.680403232574463 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.680216312408447 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.679843425750732 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.680175304412842 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.678946018218994 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.67969274520874 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.679582595825195 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.68002986907959 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.679915428161621 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.680116653442383 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.679982662200928 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.680295944213867 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.679941177368164 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.679714679718018 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.6800618171691895 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.680090427398682 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.6800456047058105 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.6803998947143555 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.679676532745361 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.679962635040283 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.6800103187561035 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.680497646331787 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.68043327331543 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.680044174194336 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.680074214935303 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.679349422454834 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.679006576538086 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.679307460784912 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.680202007293701 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.680074214935303 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.680076599121094 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.680229663848877 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.678267002105713 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.678653240203857 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.678638458251953 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.6344218254089355 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.634072303771973 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.634726524353027 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.635113716125488 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.63455057144165 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.6347174644470215 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.634566307067871 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.63478946685791 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.634678363800049 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.634609699249268 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.6348795890808105 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.634649753570557 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.634742259979248 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.63455867767334 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.634515762329102 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.634666442871094 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.634523391723633 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.634942531585693 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.634626865386963 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.6348137855529785 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.634543418884277 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.634622097015381 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.634754657745361 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.634438514709473 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.634347915649414 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.634522438049316 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.634702682495117 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.634529113769531 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.634668827056885 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.63484001159668 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.634593486785889 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.634738922119141 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.63467264175415 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.634459495544434 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.634796619415283 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.634612083435059 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.634594917297363 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.634613513946533 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.634651184082031 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.634636402130127 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.634820461273193 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.634659290313721 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.634514331817627 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.634632587432861 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.6346755027771 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.634511470794678 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.6345720291137695 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.634470462799072 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.634408473968506 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.634478569030762 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.6343913078308105 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.634519577026367 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.634640693664551 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.634561538696289 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.635130882263184 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.634726524353027 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.634790897369385 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.634706974029541 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.634881496429443 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.634670257568359 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.634647846221924 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.634418964385986 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.634365081787109 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.6348981857299805 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.634917259216309 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.634652614593506 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.634558200836182 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.6343302726745605 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.634316921234131 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.63477087020874 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.634433746337891 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.634652614593506 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.634517669677734 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.634407997131348 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.634702205657959 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.634275913238525 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.634472370147705 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.634576320648193 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.634481430053711 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.634758472442627 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.634834289550781 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.634825706481934 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.634643077850342 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.635011196136475 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.634820938110352 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.634873867034912 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.6347198486328125 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.634611129760742 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.634361743927002 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.634644508361816 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.634613037109375 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.634768009185791 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.634769439697266 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.634779930114746 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.634812355041504 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.63490104675293 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.634570598602295 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.6343865394592285 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.634805202484131 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.63482666015625 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.634646415710449 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.634552001953125 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.634678363800049 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.6356520652771 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.634761810302734 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.634861946105957 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.635007381439209 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.634713649749756 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.634528160095215 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.634710311889648 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.634641647338867 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.635028839111328 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.634829998016357 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.634672164916992 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.634788990020752 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.6344404220581055 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.634713172912598 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.634840965270996 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.634453773498535 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.6346540451049805 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.634659290313721 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.634838104248047 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.634683132171631 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.634878158569336 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.63503885269165 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.635122776031494 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.63455867767334 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.634636402130127 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.634913444519043 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.634549140930176 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.634376049041748 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.633975505828857 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.634578704833984 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.634631156921387 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.634754657745361 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.63482666015625 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.63425350189209 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.634799003601074 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.634503364562988 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.634642124176025 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.634640693664551 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.634703636169434 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.634799957275391 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.634790420532227 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.6342267990112305 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.634371757507324 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.634919166564941 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.634634017944336 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.634910583496094 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.634703636169434 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.634535789489746 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.635096073150635 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.6345977783203125 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.634106636047363 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.634876251220703 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.634749412536621 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.6347761154174805 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.634768009185791 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.635317325592041 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.635231971740723 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.634546279907227 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.634787082672119 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.634779930114746 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.634917259216309 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.635160446166992 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.63486909866333 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.634977340698242 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.634461402893066 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.634838581085205 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.634803771972656 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.634664058685303 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.635046005249023 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.634316921234131 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.634889125823975 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.63480281829834 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.634759902954102 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.6348185539245605 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.634800910949707 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.634309768676758 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.634132385253906 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.6345672607421875 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.634978294372559 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.634812355041504 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.634586811065674 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.634526252746582 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.634400367736816 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.634786128997803 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.634200572967529 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.634779453277588 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.635051250457764 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.635161876678467 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.634910583496094 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.634801387786865 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.634951114654541 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.634549617767334 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.635056018829346 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.6346917152404785 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.634525299072266 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.634586811065674 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.634840488433838 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.6347479820251465 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.634831428527832 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.635128021240234 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.634839057922363 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.634860515594482 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.6346893310546875 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.634746551513672 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.634721755981445 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.635034084320068 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.634538650512695 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.634876251220703 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.634663105010986 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.634872913360596 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.6347479820251465 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.634726524353027 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.634674072265625 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.6344122886657715 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.634836673736572 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.63470983505249 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.634652614593506 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.6350226402282715 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.634572505950928 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.634605407714844 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.6344404220581055 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.634745121002197 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.635085105895996 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.634599685668945 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.634928226470947 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.634707450866699 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.634774208068848 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.634158134460449 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.634733200073242 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.634581565856934 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.634492874145508 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.6346282958984375 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.6349592208862305 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.634838581085205 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.634437561035156 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.634930610656738 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.634708404541016 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.634586811065674 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.634941101074219 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.634531497955322 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.634833335876465 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.634850025177002 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.634787082672119 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.6345601081848145 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.634659290313721 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.63490104675293 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.634285926818848 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.634542942047119 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.634503364562988 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.634660720825195 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.634787082672119 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.634787082672119 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.634916305541992 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.634620666503906 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.634528160095215 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.634189128875732 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.634798049926758 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.634740829467773 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.634529113769531 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.634699821472168 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.634266376495361 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.63455867767334 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.634197235107422 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.634861946105957 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.634645938873291 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.634483337402344 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.634923934936523 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.634371757507324 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.634806156158447 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.6339192390441895 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.634699821472168 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.634888172149658 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.6350016593933105 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.634952068328857 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.633835792541504 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.634762763977051 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.634890556335449 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.635037899017334 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.634838581085205 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.634690284729004 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.634953498840332 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.634409427642822 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.63454008102417 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.634531021118164 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.634758949279785 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.634767532348633 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.634647846221924 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.634807109832764 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.634514331817627 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.634438514709473 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.634407997131348 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.6347575187683105 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.634664535522461 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.634851455688477 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.634836673736572 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.634371757507324 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.634819030761719 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.634323596954346 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.6348042488098145 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.634695529937744 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.634734630584717 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.634810447692871 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.634815216064453 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.634181976318359 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.634795188903809 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.6347479820251465 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.634700775146484 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.634581565856934 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.634912490844727 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.633914947509766 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.634535789489746 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.634065628051758 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.634742259979248 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.634652614593506 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.63480281829834 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.634690284729004 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.63427209854126 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.634404182434082 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.634377479553223 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.634583950042725 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.635030269622803 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.634802341461182 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.634946823120117 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.634215831756592 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.634558200836182 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.634524822235107 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.63480281829834 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.63482666015625 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.634611129760742 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.634782791137695 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.63370418548584 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.633378028869629 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.634143352508545 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.602376461029053 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.602254390716553 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.602729797363281 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.60261344909668 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.602697849273682 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.602550983428955 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.6025896072387695 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.602749824523926 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.60261344909668 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.602639198303223 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.6026740074157715 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.602715015411377 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.60275936126709 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.6026482582092285 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.602592945098877 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.602719306945801 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.602795600891113 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.602507591247559 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.602818012237549 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.602813720703125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.602723121643066 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.60258150100708 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.602563381195068 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.602481842041016 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.602636337280273 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.602618217468262 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.60267448425293 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.6028337478637695 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.6026530265808105 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.602715015411377 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.60258674621582 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.60250186920166 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.602531909942627 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.602473258972168 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.602298259735107 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.602499961853027 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.602679252624512 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.60247802734375 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.602434158325195 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.602487087249756 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.602489471435547 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.602612018585205 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.60238790512085 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.602663040161133 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.602746486663818 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.602883338928223 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.602548122406006 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.602900981903076 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.602807998657227 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.602757930755615 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.602499961853027 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.602470397949219 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.602471828460693 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.602672576904297 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.602200031280518 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.602524280548096 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.602667331695557 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.6023759841918945 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.602592468261719 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.602529525756836 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.602456569671631 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.60267448425293 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.602726936340332 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.60261344909668 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.602723121643066 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.602602958679199 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.602519989013672 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.60260009765625 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.6028876304626465 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.602643013000488 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.6025519371032715 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.602531433105469 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.602637767791748 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.602926254272461 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.602505207061768 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.602715969085693 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.602785587310791 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.602529525756836 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.602543830871582 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.602719306945801 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.60262393951416 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.6024675369262695 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.602869987487793 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.60290002822876 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.602665901184082 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.602720737457275 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.602699279785156 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.60258674621582 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.602529525756836 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.602561950683594 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.602743148803711 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.602632522583008 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.602546215057373 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.602543830871582 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.602410793304443 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.602103233337402 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.602481842041016 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.602890968322754 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.602749824523926 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.602536201477051 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.602553367614746 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.602624416351318 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.602387428283691 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.601969242095947 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.6024885177612305 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.6025309562683105 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.602753639221191 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.602603912353516 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.602726936340332 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.602846145629883 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.60233736038208 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.6020355224609375 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.602609634399414 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.602794647216797 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.602592468261719 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.6024370193481445 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.602752208709717 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.602494716644287 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.602338790893555 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.602627277374268 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.602601528167725 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.602705478668213 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.602881908416748 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.602294921875 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.602343559265137 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.602423191070557 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.602542877197266 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.602538108825684 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.602520942687988 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.602532863616943 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.6027631759643555 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.6031413078308105 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.6026482582092285 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.60277795791626 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.602671146392822 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.602450847625732 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.602741718292236 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.60235071182251 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.602287292480469 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.6026482582092285 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.602664947509766 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.602461338043213 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.602402687072754 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.6024627685546875 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.602417469024658 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.602417469024658 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.602536201477051 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.602646827697754 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.602489948272705 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.602656364440918 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.602520942687988 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.602190971374512 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.602662086486816 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.602461338043213 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.602517127990723 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.602338790893555 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.602684020996094 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.602309703826904 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.60207462310791 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.602107524871826 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.602138519287109 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.6025800704956055 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.6025710105896 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.602563381195068 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.60262393951416 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.602276802062988 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.601959228515625 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.6025309562683105 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.602711200714111 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.602665901184082 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.6026506423950195 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.602342128753662 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.602908134460449 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.602299690246582 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.601961135864258 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.602397441864014 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.602447986602783 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.602572441101074 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.60263729095459 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.602451801300049 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.602328300476074 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.602071762084961 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.60259485244751 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.602668285369873 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.6024370193481445 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.602582931518555 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.602163791656494 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.602546691894531 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.601992130279541 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.602412700653076 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.6025390625 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.602677822113037 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.602524757385254 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.602311611175537 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.602516174316406 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.601955413818359 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.602611541748047 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.602670669555664 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.602784156799316 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.602000713348389 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.602520942687988 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.6022515296936035 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.60172176361084 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.602502822875977 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.6024675369262695 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.602555751800537 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.602536201477051 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.6023688316345215 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.602229118347168 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.602573871612549 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.602676868438721 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.602524280548096 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.602542877197266 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.602742671966553 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.6023149490356445 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.602523326873779 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.602014541625977 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.602483749389648 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.602370738983154 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.602789402008057 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.602116107940674 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.602711200714111 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.602266311645508 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.602450847625732 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.602467060089111 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.602459907531738 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.602584362030029 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.602471828460693 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.602449417114258 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.602384567260742 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.602477550506592 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.60264778137207 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.6024041175842285 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.602625846862793 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.602574825286865 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.602072715759277 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.602046489715576 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.6019792556762695 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.60269021987915 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.602668285369873 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.602512359619141 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.602426528930664 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.602242469787598 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.602358341217041 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.6020188331604 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.602476119995117 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.602467060089111 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.602512359619141 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.602453231811523 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.602333068847656 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.602257251739502 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.601929664611816 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.602769374847412 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.602492332458496 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.602462291717529 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.602294445037842 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.6022443771362305 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.602373123168945 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.603030204772949 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.602533340454102 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.602581977844238 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.602468013763428 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.60261344909668 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.602210521697998 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.602398872375488 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.602407932281494 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.602467060089111 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.602761745452881 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.602672100067139 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.602674961090088 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.602019309997559 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.602397441864014 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.602300643920898 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.602636337280273 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.602447986602783 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.602422714233398 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.602384090423584 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.602832794189453 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.602330207824707 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.601839065551758 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.6024250984191895 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.602708339691162 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.602721214294434 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.602322101593018 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.60220193862915 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.602118492126465 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.602388381958008 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.602651596069336 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.602596759796143 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.60247278213501 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.602462291717529 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.602044582366943 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.602418899536133 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.6025166511535645 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.602704048156738 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.602518558502197 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.602612018585205 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.602584362030029 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.601989269256592 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.602135181427002 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.602342128753662 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.602805137634277 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.60256814956665 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.602594375610352 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.602599620819092 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.602038383483887 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.602548122406006 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.602119445800781 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.602499961853027 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.602678298950195 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.6025471687316895 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.602494239807129 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.6028971672058105 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.602340221405029 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.602397441864014 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.602656364440918 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.602447986602783 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.602482795715332 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.602871894836426 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.602436065673828 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.602344512939453 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.60256814956665 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.602691173553467 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.602657794952393 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.602652549743652 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.602514743804932 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.601973533630371 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.602214336395264 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.602125644683838 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.602717876434326 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.6026482582092285 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.602652549743652 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.6025166511535645 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.601644992828369 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.601531982421875 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.602085590362549 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.571828842163086 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.571781158447266 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.572113037109375 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.572244644165039 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.5720720291137695 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.57233190536499 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.571810245513916 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.5720930099487305 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.571967601776123 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.5720977783203125 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.572041034698486 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.572040557861328 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.572009086608887 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.5718994140625 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.572058200836182 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.572035312652588 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.572187900543213 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.571959972381592 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.571848392486572 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.5721564292907715 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.57219934463501 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.572021007537842 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.572030544281006 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.571839809417725 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.571844577789307 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.571913719177246 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.57194709777832 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.571967124938965 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.5719804763793945 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.572074890136719 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.572109222412109 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.572021484375 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.572043418884277 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.57192850112915 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.571994781494141 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.572033405303955 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.572225093841553 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.572134017944336 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.57198429107666 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.572074890136719 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.572048187255859 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.571883201599121 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.572117328643799 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.572025299072266 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.571937561035156 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.571788787841797 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.571893692016602 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.571908473968506 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.571981906890869 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.571682453155518 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.5720720291137695 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.572030544281006 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.571884632110596 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.571943283081055 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.572483062744141 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.571987628936768 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.571972370147705 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.5721845626831055 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.572361469268799 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.572437286376953 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.57222843170166 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.572386741638184 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.57194185256958 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.572229862213135 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.572159290313721 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.572035312652588 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.572171688079834 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.571986675262451 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.571958065032959 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.571884632110596 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.572030067443848 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.57206916809082 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.572172164916992 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.571891784667969 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.571754455566406 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.571744918823242 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.57232141494751 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.571944713592529 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.572050094604492 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.572016716003418 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.572356224060059 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.572178363800049 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.572030544281006 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.572007656097412 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.572020053863525 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.572103977203369 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.572139739990234 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.572005748748779 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.571561813354492 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.572153568267822 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.572139739990234 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.57211446762085 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.572234153747559 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.572141647338867 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.571890830993652 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.572420120239258 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.572018623352051 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.57173490524292 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.572118759155273 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.572028160095215 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.572018623352051 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.571844577789307 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.572028160095215 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.5721917152404785 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.5720696449279785 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.5720930099487305 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.5720720291137695 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.5721354484558105 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.5721049308776855 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.571959972381592 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.572145462036133 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.572502613067627 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.572106838226318 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.572144031524658 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.571915149688721 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.57220458984375 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.571643352508545 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.572196960449219 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.571771621704102 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.572145938873291 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.572014808654785 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.571995735168457 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.571767330169678 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.5720014572143555 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.572246551513672 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.572083950042725 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.572061538696289 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.572113037109375 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.572052001953125 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.571728229522705 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.57204008102417 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.571646213531494 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.571781158447266 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.571721076965332 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.57204532623291 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.572025299072266 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.5719099044799805 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.572187900543213 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.572551250457764 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.571793556213379 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.571654796600342 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.572011947631836 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.572225093841553 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.572052001953125 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.571558952331543 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.571587085723877 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.571832656860352 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.57208251953125 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.572052955627441 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.5718488693237305 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.571791648864746 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.572105407714844 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.571842193603516 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.5719733238220215 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.572029113769531 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.572202205657959 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.572006702423096 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.572016716003418 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.57184362411499 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.572312355041504 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.572051525115967 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.572081089019775 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.572076797485352 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.57209587097168 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.571831703186035 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.571763038635254 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.572098731994629 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.571634292602539 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.572237968444824 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.572146892547607 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.572061538696289 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.571903228759766 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.571811199188232 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.572035312652588 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.572342395782471 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.571959018707275 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.57207727432251 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.572031497955322 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.5720720291137695 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.571783065795898 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.57197904586792 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.572105407714844 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.572081089019775 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.571854114532471 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.572086811065674 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.571735858917236 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.572086811065674 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.571916580200195 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.5718913078308105 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.572117805480957 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.572498321533203 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.571965217590332 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.572225093841553 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.5719733238220215 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.572102069854736 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.572369575500488 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.572068691253662 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.571864604949951 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.572117328643799 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.571882724761963 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.571958065032959 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.572301864624023 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.5719828605651855 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.572214603424072 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.571887969970703 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.572015285491943 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.5720930099487305 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.57201623916626 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.571741104125977 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.571927070617676 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.57224178314209 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.572234153747559 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.5719218254089355 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.571877479553223 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.572021484375 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.572018623352051 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.571866989135742 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.5721821784973145 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.57202672958374 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.571925163269043 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.572504997253418 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.5717363357543945 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.571770668029785 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.572320938110352 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.572089195251465 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.57217264175415 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.571986675262451 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.57218074798584 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.571704387664795 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.572010517120361 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.571958065032959 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.571962356567383 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.571903228759766 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.571988582611084 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.571802139282227 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.571993827819824 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.572134971618652 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.571990966796875 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.572141170501709 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.572136878967285 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.571974754333496 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.572311878204346 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.57203483581543 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.572552680969238 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.571994781494141 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.571901321411133 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.572122097015381 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.572051048278809 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.572183609008789 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.571717262268066 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.571977138519287 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.571994781494141 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.57194185256958 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.572216510772705 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.571872711181641 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.572206020355225 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.571887493133545 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.571848392486572 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.571322441101074 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.572097301483154 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.57217264175415 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.5719404220581055 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.572115898132324 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.571890354156494 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.572282314300537 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.5716071128845215 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.572154521942139 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.571961402893066 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.57194185256958 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.57198429107666 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.5716376304626465 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.572134017944336 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.572011947631836 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.571913242340088 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.572141647338867 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.571932315826416 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.572206020355225 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.571577072143555 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.571864128112793 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.572083950042725 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.572046756744385 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.572065353393555 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.571944713592529 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.572115898132324 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.571653366088867 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.572117328643799 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.571706295013428 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.572188854217529 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.572168827056885 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.57211446762085 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.572205543518066 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.572141170501709 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.571990013122559 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.571651458740234 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.571953773498535 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.572092056274414 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.572152137756348 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.57219934463501 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.572172164916992 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.57191276550293 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.571901798248291 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.57200288772583 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.572081089019775 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.572021007537842 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.572029113769531 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.5720906257629395 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.5719194412231445 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.5717973709106445 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.571866035461426 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.57200813293457 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.57212495803833 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.571962833404541 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.5714192390441895 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.572219371795654 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.5719780921936035 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.572021007537842 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.572081089019775 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.572014808654785 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.572044849395752 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.57178258895874 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.571817398071289 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.571791648864746 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.572033405303955 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.572109222412109 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.572051048278809 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.571932315826416 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.571932315826416 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.571797847747803 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.571959018707275 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.571951866149902 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.572109222412109 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.572110176086426 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.5720109939575195 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.57110595703125 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.571197986602783 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.571202278137207 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.549679756164551 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.549529552459717 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.550046920776367 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.549909591674805 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.54995584487915 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.549922466278076 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.5499138832092285 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.550005912780762 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.5499653816223145 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.549902439117432 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.549901008605957 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.550042629241943 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.550070762634277 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.549787521362305 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.54984712600708 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.549759864807129 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.550008296966553 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.550016403198242 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.549990653991699 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.549994945526123 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.55000114440918 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.549890518188477 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.549831867218018 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.549957275390625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.549963474273682 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.550020694732666 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.550027847290039 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.5499701499938965 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.55001163482666 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.550021648406982 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.549990177154541 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.549970626831055 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.5498127937316895 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.549901008605957 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.549922466278076 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.550014019012451 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.549872398376465 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.549934387207031 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.550054550170898 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.549704074859619 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.549966335296631 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.5496721267700195 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.549783706665039 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.550055980682373 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.54986047744751 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.550090312957764 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.549957275390625 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.549729347229004 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.549787521362305 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.54981803894043 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.549890518188477 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.54983377456665 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.549961566925049 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.5498948097229 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.549497604370117 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.5500922203063965 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.54985237121582 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.549903869628906 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.549909591674805 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.549835205078125 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.549649715423584 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.5498762130737305 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.549607753753662 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.550161838531494 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.549993991851807 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.54979944229126 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.549777030944824 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.550101280212402 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.549993515014648 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.5499396324157715 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.550195693969727 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.549893379211426 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.549985885620117 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.549848556518555 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.549648284912109 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.5498247146606445 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.54979944229126 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.5497660636901855 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.5499348640441895 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.5498456954956055 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.549688339233398 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.549657821655273 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.549929141998291 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.549681663513184 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.5498762130737305 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.55012321472168 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.550038814544678 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.549957275390625 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.550027847290039 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.550025939941406 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.550160884857178 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.549939155578613 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.549936771392822 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.549910545349121 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.549982070922852 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.549294471740723 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.549777984619141 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.550477504730225 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.549796104431152 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.5498151779174805 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.550048351287842 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.549872398376465 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.549821853637695 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.549640655517578 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.549787521362305 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.549920082092285 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.54983377456665 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.549920082092285 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.549806118011475 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.549665927886963 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.549813270568848 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.549261093139648 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.550054550170898 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.550029277801514 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.550193786621094 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.5498576164245605 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.549993515014648 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.549598693847656 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.550041675567627 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.549729347229004 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.549861907958984 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.54993200302124 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.550078392028809 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.54966926574707 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.549963474273682 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.5496439933776855 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.549987316131592 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.549814224243164 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.5499467849731445 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.549997806549072 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.549429893493652 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.550318241119385 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.549612522125244 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.5500335693359375 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.549831867218018 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.549816608428955 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.550042629241943 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.549982070922852 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.549505710601807 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.550147533416748 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.550008296966553 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.549954414367676 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.549981594085693 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.549707889556885 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.5502471923828125 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.550253868103027 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.549620151519775 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.549987316131592 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.549789905548096 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.550118446350098 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.549960136413574 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.54958963394165 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.549703598022461 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.550120830535889 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.549902439117432 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.549824237823486 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.550081729888916 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.549829959869385 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.549994468688965 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.5495524406433105 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.549531936645508 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.550014019012451 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.5500593185424805 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.550126075744629 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.55001163482666 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.549798011779785 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.549458026885986 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.549938201904297 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.550122261047363 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.549778461456299 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.549924850463867 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.550048351287842 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.5502119064331055 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.550030708312988 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.54971981048584 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.5500078201293945 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.549768924713135 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.549901008605957 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.549946308135986 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.549938201904297 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.549842357635498 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.549345016479492 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.550093173980713 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.549874782562256 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.5497965812683105 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.550013065338135 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.549510955810547 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.549774169921875 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.549862384796143 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.549934387207031 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.550008296966553 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.549877166748047 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.549856185913086 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.549891948699951 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.54966926574707 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.5496625900268555 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.54977560043335 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.549820423126221 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.550016403198242 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.549826145172119 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.550395965576172 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.550048351287842 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.549854278564453 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.549950122833252 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.549835681915283 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.5498127937316895 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.549778461456299 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.549882888793945 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.549938201904297 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.550137996673584 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.549839496612549 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.549893379211426 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.550008296966553 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.549924850463867 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.549868106842041 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.549898624420166 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.5498127937316895 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.549840450286865 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.549670696258545 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.550047397613525 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.549597263336182 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.550177574157715 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.5498833656311035 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.5494489669799805 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.549778938293457 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.550057888031006 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.550052165985107 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.549875259399414 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.550024509429932 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.550101280212402 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.549591541290283 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.549835205078125 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.549718856811523 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.5499677658081055 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.549976825714111 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.549520015716553 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.549643516540527 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.5497822761535645 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.549840450286865 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.550076961517334 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.5497145652771 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.549959182739258 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.549525737762451 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.549461364746094 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.5493268966674805 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.549930572509766 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.549983024597168 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.549835205078125 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.549956321716309 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.550193786621094 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.549629211425781 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.549497604370117 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.549896240234375 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.550004005432129 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.550067901611328 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.550020217895508 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.549963474273682 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.55003547668457 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.54994010925293 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.54995584487915 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.54987096786499 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.549812316894531 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.550018787384033 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.549551010131836 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.549985885620117 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.550036430358887 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.549935340881348 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.550036430358887 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.549789905548096 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.550091743469238 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.550062656402588 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.549594402313232 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.549846649169922 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.5498456954956055 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.549856185913086 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.549945831298828 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.549898147583008 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.549707889556885 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.54976224899292 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.5496673583984375 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.549899101257324 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.549837112426758 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.549866676330566 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.54994010925293 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.549661159515381 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.549443244934082 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.549942493438721 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.549980640411377 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.549829006195068 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.549872398376465 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.549946308135986 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.549902439117432 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.549468994140625 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.549771785736084 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.550018310546875 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.549909591674805 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.550011157989502 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.549882888793945 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.549427032470703 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.549561500549316 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.5497212409973145 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.550093650817871 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.549999237060547 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.54990816116333 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.549953460693359 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.549560546875 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.549743175506592 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.549705505371094 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.549818992614746 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.549872398376465 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.5497636795043945 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.550017356872559 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.55030632019043 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.549476623535156 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.549694538116455 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.550054550170898 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.549952983856201 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.550017833709717 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.550148010253906 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.549736022949219 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.549929141998291 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.5499796867370605 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.550221920013428 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.550118923187256 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.550044059753418 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.550078868865967 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.549501895904541 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.5493974685668945 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.549342155456543 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.549901962280273 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.549964904785156 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.549829959869385 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.5501580238342285 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.549219608306885 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.5497822761535645 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.54974365234375 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.526448726654053 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.526580333709717 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.526686191558838 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.526494026184082 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.5268473625183105 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.526706218719482 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.526655197143555 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.526659965515137 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.526793479919434 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.526744842529297 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.5268754959106445 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.526676177978516 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.526729106903076 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.526602745056152 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.526669025421143 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.5269455909729 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.5269083976745605 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.526833534240723 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.526732921600342 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.5269246101379395 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.526805400848389 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.526806831359863 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.5269293785095215 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.526817321777344 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.526881694793701 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.526762008666992 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.526730537414551 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.526660919189453 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.526641845703125 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.526825428009033 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.526707649230957 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.526920318603516 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.526658535003662 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.526756763458252 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.526644706726074 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.526697158813477 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.52687406539917 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.526844024658203 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.5267653465271 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.526700973510742 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.526749134063721 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.526762008666992 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.526646137237549 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.526730537414551 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.52666711807251 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.526564598083496 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.526747226715088 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.5265350341796875 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.5264763832092285 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.526648998260498 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.526609420776367 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.526767253875732 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.526482582092285 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.526932239532471 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.527075290679932 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.526613235473633 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.526548385620117 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.526632785797119 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.526899337768555 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.526902198791504 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.5268354415893555 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.526819229125977 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.526625633239746 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.526825428009033 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.5266432762146 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.526731491088867 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.52656888961792 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.526235103607178 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.526689052581787 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.526713848114014 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.526739597320557 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.526637554168701 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.526651382446289 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.526546955108643 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.526556491851807 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.526426315307617 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.526785850524902 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.5267462730407715 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.526516914367676 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.526642322540283 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.526956081390381 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.526688098907471 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.526791095733643 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.526520252227783 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.526695251464844 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.52676248550415 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.526717185974121 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.526615619659424 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.526389122009277 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.526676177978516 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.526672840118408 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.526773929595947 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.526905536651611 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.526733875274658 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.526585578918457 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.527333736419678 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.526914596557617 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.526303768157959 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.526667594909668 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.526823997497559 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.526660919189453 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.526721000671387 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.5266289710998535 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.527103424072266 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.526858329772949 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.526485443115234 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.526750564575195 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.526634693145752 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.526729106903076 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.526713848114014 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.526908874511719 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.52709436416626 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.526732921600342 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.526631832122803 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.526490211486816 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.526642322540283 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.526485443115234 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.526991367340088 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.5263590812683105 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.526762008666992 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.526621341705322 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.526717662811279 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.5262298583984375 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.526436805725098 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.52683687210083 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.52673864364624 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.526431560516357 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.526577949523926 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.526615619659424 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.526637077331543 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.526759624481201 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.526488304138184 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.526749134063721 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.526581287384033 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.526651382446289 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.526712894439697 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.526271820068359 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.5267133712768555 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.527283668518066 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.526320457458496 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.526630401611328 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.526856899261475 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.526518821716309 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.526792049407959 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.526251792907715 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.526307582855225 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.5266499519348145 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.526589393615723 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.5265021324157715 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.526838302612305 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.526430130004883 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.526719570159912 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.526674747467041 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.52682638168335 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.526820182800293 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.526856899261475 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.526753902435303 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.526599884033203 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.526368618011475 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.52689790725708 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.526917457580566 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.52665376663208 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.526588439941406 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.526792526245117 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.526491641998291 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.526638984680176 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.526823997497559 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.526756763458252 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.5266499519348145 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.52667760848999 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.526572227478027 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.526630401611328 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.526406288146973 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.52685022354126 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.526396751403809 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.526601314544678 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.526717185974121 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.526833534240723 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.526568412780762 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.526453971862793 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.52648401260376 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.526730537414551 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.526534557342529 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.526549816131592 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.526722431182861 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.526566505432129 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.526316165924072 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.526669979095459 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.526523590087891 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.526545524597168 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.526839733123779 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.526744365692139 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.526615619659424 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.5265960693359375 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.526933670043945 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.526771545410156 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.5267252922058105 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.526702880859375 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.527001857757568 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.526562690734863 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.526426315307617 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.526558876037598 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.526411533355713 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.526622295379639 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.526684761047363 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.526835918426514 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.526825428009033 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.526782035827637 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.5265583992004395 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.526557922363281 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.526697158813477 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.526755332946777 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.526587009429932 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.526575088500977 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.526525974273682 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.526732921600342 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.526357650756836 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.526784420013428 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.526619911193848 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.526336669921875 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.5268683433532715 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.526219367980957 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.526854515075684 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.526947975158691 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.5267534255981445 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.526663303375244 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.526575088500977 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.526771545410156 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.526385307312012 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.5265631675720215 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.52659273147583 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.526658058166504 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.526648998260498 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.526669979095459 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.526342868804932 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.52672815322876 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.526853561401367 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.526707649230957 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.526639461517334 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.526607990264893 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.526583671569824 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.526543617248535 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.526739597320557 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.52667760848999 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.526750564575195 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.526451110839844 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.526530742645264 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.526566028594971 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.526686191558838 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.526422023773193 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.5267744064331055 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.526607990264893 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.526777744293213 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.526732921600342 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.526619911193848 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.526597499847412 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.526237487792969 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.5262579917907715 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.526718616485596 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.526476860046387 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.526723861694336 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.526737213134766 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.526578903198242 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.526398181915283 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.52672815322876 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.526199817657471 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.526570796966553 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.5266642570495605 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.526486873626709 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.526647567749023 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.526126384735107 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.526575565338135 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.5261030197143555 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.52653169631958 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.52665376663208 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.5266194343566895 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.526805400848389 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.526578903198242 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.526644229888916 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.526477813720703 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.526679515838623 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.5266923904418945 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.52646017074585 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.5268235206604 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.526465892791748 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.5266923904418945 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.526549339294434 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.526631832122803 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.52682638168335 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.526707649230957 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.526906490325928 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.526345729827881 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.526574611663818 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.52653169631958 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.526632785797119 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.526653289794922 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.5266923904418945 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.526839733123779 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.526739120483398 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.526550769805908 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.526568412780762 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.526575088500977 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.526772499084473 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.526651382446289 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.526676177978516 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.526638984680176 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.526443958282471 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.526534557342529 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.526747226715088 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.526637554168701 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.526768684387207 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.526652812957764 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.526336669921875 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.526716232299805 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.526465892791748 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.5267791748046875 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.526700019836426 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.5266923904418945 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.526498794555664 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.526665210723877 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.526490688323975 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.526273250579834 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.52650785446167 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.526619911193848 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.526750564575195 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.5267133712768555 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.526564598083496 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.526453971862793 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.526618003845215 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.5265350341796875 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.52687406539917 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.526861667633057 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.526642322540283 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.525847434997559 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.525552749633789 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.52618932723999 + } + ] + } + ] +} \ No newline at end of file