diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json"
new file mode 100644--- /dev/null
+++ "b/reduced_ppl_average.json"
@@ -0,0 +1,11840 @@
+{
+    "original_ppl": 9.97196102142334,
+    "reduced_ppl": [
+        {
+            "bit": 1,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 83332.109375
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 81348.953125
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 25781.796875
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 27834.796875
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 62059.78125
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 64951.6875
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 62222.87109375
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 85301.765625
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 86110.703125
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 45133.1328125
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 36500.11328125
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 77381.359375
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 49991.375
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 46378.9609375
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 88139.484375
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 90501.71875
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 120570.203125
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 50978.390625
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 78643.28125
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 82742.3125
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 64036.484375
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 76686.3828125
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 81167.9296875
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 87316.3125
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 60217.55078125
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 59247.91015625
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 62741.40234375
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 84264.7265625
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 72960.6640625
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 93884.59375
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 77069.7578125
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 92530.4609375
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 101563.9921875
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 50916.39453125
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 89985.078125
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 86380.2421875
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 68246.171875
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 97836.4609375
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 67856.2578125
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 92100.484375
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 100485.90625
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 41168.40234375
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 79279.359375
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 82641.6796875
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 63274.9296875
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 52274.33203125
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 60532.96875
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 100568.0625
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 78218.3515625
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 88200.6171875
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 88442.1875
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 104099.015625
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 139569.671875
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 53775.09375
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 56642.953125
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 70138.421875
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 76445.34375
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 82595.5078125
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 108704.234375
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 76762.40625
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 90175.453125
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 60534.23828125
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 52055.08984375
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 83108.46875
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 83471.1484375
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 136432.921875
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 152531.609375
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 74755.015625
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 76557.9921875
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 133054.65625
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 97931.8671875
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 91111.8046875
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 70577.2421875
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 34539.39453125
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 78766.8984375
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 93544.703125
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 71034.9375
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 91325.03125
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 85825.4765625
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 79586.921875
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 82422.2421875
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 51500.8125
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 67704.21875
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 64296.7421875
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 85187.3828125
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 85370.0546875
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 45587.73828125
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 45965.9296875
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 97300.6953125
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 77209.828125
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 58815.9453125
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 85948.0078125
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 79145.953125
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 51608.9296875
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 85309.3359375
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 82983.8125
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 69577.9375
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 76646.3125
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 86569.8359375
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 86495.3203125
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 78097.0078125
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 89474.8125
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 62052.5
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 121601.6328125
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 65683.3515625
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 90000.015625
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 85875.5
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 51253.1796875
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 60614.70703125
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 69880.375
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 82342.890625
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 77203.5703125
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 86468.59375
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 85792.90625
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 55330.50390625
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 78712.8359375
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 86215.640625
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 102103.765625
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 82107.328125
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 81355.1640625
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 85610.15625
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 73543.0
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 56332.01171875
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 84936.15625
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 80587.7578125
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 76033.25
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 85493.8125
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 89435.65625
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 94210.34375
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 56984.73046875
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 79825.453125
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 59850.96484375
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 87420.8828125
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 87652.3828125
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 83483.0859375
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 103584.7421875
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 56013.58984375
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 100148.1875
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 91430.125
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 84644.1640625
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 86207.0078125
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 85745.3046875
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 103126.3125
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 79577.7421875
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 87526.578125
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 89423.28125
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 82647.515625
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 85887.8671875
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 82824.2578125
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 104417.4765625
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 69490.734375
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 101418.03125
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 85467.7265625
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 73187.4296875
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 85862.8828125
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 86954.09375
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 88807.8125
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 70540.1015625
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 67607.828125
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 74690.4453125
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 99885.3125
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 92043.3125
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 82548.1015625
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 74932.453125
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 68666.203125
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 83727.390625
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 83494.7109375
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 100388.8671875
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 86749.25
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 83572.3125
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 82877.3515625
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 89907.109375
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 85133.375
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 94788.8359375
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 84814.09375
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 83655.2421875
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 85422.8203125
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 103965.4765625
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 86323.90625
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 101230.765625
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 83441.78125
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 92780.625
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 85717.34375
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 85034.953125
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 85586.15625
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 75896.609375
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 87989.3984375
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 78273.125
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 88496.859375
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 86038.546875
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 84773.5
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 89991.34375
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 78720.7890625
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 108667.953125
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 89630.0703125
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 78953.484375
+                }
+            ]
+        },
+        {
+            "bit": 2,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 168.69410705566406
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 167.3025665283203
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 134.85409545898438
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 147.0169219970703
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 153.91915893554688
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 157.78953552246094
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 153.5771484375
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 168.2405548095703
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 168.6426239013672
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 163.9848175048828
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 175.6777801513672
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 158.56808471679688
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 169.42819213867188
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 162.0041046142578
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 168.2506561279297
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 168.59390258789062
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 158.20021057128906
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 152.7701416015625
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 144.7792510986328
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 168.1245880126953
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 166.7340087890625
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 167.2913055419922
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 169.46438598632812
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 159.04104614257812
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 158.66580200195312
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 150.30787658691406
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 164.19912719726562
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 160.14694213867188
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 163.7737579345703
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 168.6818084716797
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 169.0232696533203
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 163.02841186523438
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 152.95265197753906
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 165.4967498779297
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 173.02493286132812
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 169.71926879882812
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 168.32321166992188
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 171.78506469726562
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 172.12879943847656
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 156.8002471923828
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 169.80587768554688
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 168.16331481933594
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 171.93313598632812
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 170.42822265625
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 166.3150634765625
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 164.08706665039062
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 159.70425415039062
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 165.3900909423828
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 162.13487243652344
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 170.3074188232422
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 168.4875030517578
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 163.2733917236328
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 156.7130889892578
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 150.77174377441406
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 159.7360076904297
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 165.3878936767578
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 173.43414306640625
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 169.6393280029297
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 167.9151611328125
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 167.08619689941406
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 160.76515197753906
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 162.380126953125
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 165.14808654785156
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 168.6042022705078
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 167.47824096679688
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 168.47898864746094
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 163.73831176757812
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 166.50613403320312
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 165.7300262451172
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 168.846923828125
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 168.58209228515625
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 167.91220092773438
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 174.09750366210938
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 167.61534118652344
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 169.89886474609375
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 169.2921142578125
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 169.01593017578125
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 167.5283966064453
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 168.47947692871094
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 165.90406799316406
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 165.99972534179688
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 165.912841796875
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 172.89553833007812
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 170.34291076660156
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 170.30311584472656
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 168.70327758789062
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 169.1955108642578
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 165.52769470214844
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 167.78005981445312
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 172.9617462158203
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 170.7941436767578
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 169.14678955078125
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 167.37380981445312
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 169.5671844482422
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 164.41168212890625
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 162.4380645751953
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 170.59213256835938
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 176.98516845703125
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 169.9107666015625
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 168.72178649902344
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 168.07215881347656
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 170.56326293945312
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 169.74880981445312
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 170.60238647460938
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 168.23294067382812
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 168.81463623046875
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 168.92770385742188
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 169.9482879638672
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 166.65753173828125
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 167.7421417236328
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 170.9343719482422
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 166.71420288085938
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 168.89065551757812
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 169.56088256835938
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 170.41140747070312
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 167.99749755859375
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 166.1726837158203
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 169.96035766601562
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 167.08116149902344
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 169.047119140625
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 167.61502075195312
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 167.5784149169922
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 175.95620727539062
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 171.89068603515625
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 167.10482788085938
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 166.0119171142578
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 169.005859375
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 170.59197998046875
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 168.8288116455078
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 166.205810546875
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 166.53225708007812
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 167.81149291992188
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 171.29623413085938
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 170.4592742919922
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 169.1224365234375
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 174.17156982421875
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 171.35145568847656
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 168.1353302001953
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 167.7538299560547
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 168.72451782226562
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 169.44894409179688
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 169.5083465576172
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 168.2606201171875
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 169.7484130859375
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 169.15194702148438
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 168.19691467285156
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 167.42849731445312
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 169.51043701171875
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 169.20535278320312
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 168.32289123535156
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 173.6912841796875
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 168.3231201171875
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 167.8394317626953
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 169.46389770507812
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 169.63949584960938
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 169.69911193847656
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 168.16082763671875
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 165.98532104492188
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 167.897705078125
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 168.83944702148438
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 166.7910919189453
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 169.135498046875
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 167.2165069580078
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 168.8903350830078
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 167.9126739501953
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 167.8650360107422
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 168.4675750732422
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 166.85760498046875
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 169.2283477783203
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 168.97378540039062
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 167.07391357421875
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 167.07662963867188
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 167.44293212890625
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 169.12307739257812
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 167.47384643554688
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 169.68487548828125
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 168.2967987060547
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 171.37777709960938
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 170.510498046875
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 168.1109619140625
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 168.1544189453125
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 168.1858367919922
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 169.4444122314453
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 167.9197998046875
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 168.7620086669922
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 168.62879943847656
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 168.68331909179688
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 167.77685546875
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 169.30543518066406
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 169.27587890625
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 168.8433074951172
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 169.55361938476562
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 167.65817260742188
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 165.5074920654297
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 167.4623565673828
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 165.63079833984375
+                }
+            ]
+        },
+        {
+            "bit": 3,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 16.816091537475586
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 16.80735969543457
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 16.645936965942383
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 16.93777847290039
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 16.7054443359375
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 16.74637794494629
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 16.680191040039062
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 16.808029174804688
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 16.817955017089844
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 16.794597625732422
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 17.078054428100586
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 16.684486389160156
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 16.77771759033203
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 16.807056427001953
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 16.829391479492188
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 16.85578155517578
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 16.81163215637207
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 16.761272430419922
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 16.623249053955078
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 16.745830535888672
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 16.735363006591797
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 16.834251403808594
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 16.811296463012695
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 16.80232810974121
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 16.77528953552246
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 16.66208267211914
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 16.772022247314453
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 16.740541458129883
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 16.826749801635742
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 16.81400489807129
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 16.795495986938477
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 16.771345138549805
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 16.66862678527832
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 16.704448699951172
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 16.728321075439453
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 16.83759307861328
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 16.834495544433594
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 16.800758361816406
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 16.80919647216797
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 16.59544563293457
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 16.76308250427246
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 16.8048038482666
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 16.914257049560547
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 16.84715461730957
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 16.825233459472656
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 16.83127212524414
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 16.668018341064453
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 16.790830612182617
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 16.75554084777832
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 16.834848403930664
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 16.82583236694336
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 16.790273666381836
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 16.79183578491211
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 16.564863204956055
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 16.64543342590332
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 16.75745391845703
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 16.867660522460938
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 16.824426651000977
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 16.823348999023438
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 16.83637809753418
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 16.6285400390625
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 16.746749877929688
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 16.785511016845703
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 16.826574325561523
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 16.831188201904297
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 16.818683624267578
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 16.809307098388672
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 16.768367767333984
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 16.749040603637695
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 16.7552433013916
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 16.843311309814453
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 16.82830047607422
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 16.85602569580078
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 16.78997039794922
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 16.79315757751465
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 16.79081153869629
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 16.817453384399414
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 16.802064895629883
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 16.813669204711914
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 16.777050018310547
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 16.724058151245117
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 16.715961456298828
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 16.78472328186035
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 16.784912109375
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 16.84333038330078
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 16.839614868164062
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 16.82872772216797
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 16.828916549682617
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 16.753414154052734
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 16.804367065429688
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 16.81096649169922
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 16.81706428527832
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 16.80708885192871
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 16.78807258605957
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 16.7968807220459
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 16.651710510253906
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 16.751850128173828
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 16.83733367919922
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 16.855628967285156
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 16.83863639831543
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 16.828649520874023
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 16.83321762084961
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 16.75590705871582
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 16.78203010559082
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 16.800777435302734
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 16.81707000732422
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 16.81329345703125
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 16.830177307128906
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 16.814172744750977
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 16.749544143676758
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 16.782508850097656
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 16.79301643371582
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 16.821210861206055
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 16.835525512695312
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 16.794036865234375
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 16.860326766967773
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 16.795434951782227
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 16.793045043945312
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 16.842557907104492
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 16.829526901245117
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 16.826160430908203
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 16.854816436767578
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 16.840280532836914
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 16.797454833984375
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 16.827211380004883
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 16.821142196655273
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 16.816619873046875
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 16.832456588745117
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 16.801607131958008
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 16.806598663330078
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 16.775604248046875
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 16.771047592163086
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 16.798187255859375
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 16.866594314575195
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 16.800806045532227
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 16.85487174987793
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 16.834245681762695
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 16.771350860595703
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 16.776689529418945
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 16.772886276245117
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 16.834583282470703
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 16.827171325683594
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 16.811311721801758
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 16.786184310913086
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 16.795663833618164
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 16.79833221435547
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 16.78153419494629
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 16.836599349975586
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 16.819507598876953
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 16.8144588470459
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 16.851163864135742
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 16.78438377380371
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 16.80192756652832
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 16.793245315551758
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 16.824665069580078
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 16.825010299682617
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 16.813615798950195
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 16.798166275024414
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 16.749866485595703
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 16.7833309173584
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 16.74230194091797
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 16.831605911254883
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 16.823076248168945
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 16.813953399658203
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 16.849159240722656
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 16.76750373840332
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 16.77958869934082
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 16.73868179321289
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 16.82379722595215
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 16.818166732788086
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 16.821287155151367
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 16.840299606323242
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 16.7641544342041
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 16.766313552856445
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 16.754161834716797
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 16.83121109008789
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 16.81581687927246
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 16.803930282592773
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 16.8447265625
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 16.75250816345215
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 16.766193389892578
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 16.7667293548584
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 16.832719802856445
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 16.822650909423828
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 16.823837280273438
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 16.85397720336914
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 16.7725772857666
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 16.75528335571289
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 16.76927947998047
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 16.828399658203125
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 16.81958770751953
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 16.833404541015625
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 16.843053817749023
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 16.69053840637207
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 16.710926055908203
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 16.69918441772461
+                }
+            ]
+        },
+        {
+            "bit": 4,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 12.195830345153809
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 12.198372840881348
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 12.180115699768066
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 12.190705299377441
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 12.178506851196289
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 12.185070991516113
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 12.172568321228027
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 12.194560050964355
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 12.19727611541748
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 12.191821098327637
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 12.202656745910645
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 12.166489601135254
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 12.192140579223633
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 12.187891960144043
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 12.200326919555664
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 12.192219734191895
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 12.193717002868652
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 12.198744773864746
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 12.174346923828125
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 12.183810234069824
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 12.186564445495605
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 12.199477195739746
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 12.191658020019531
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 12.189967155456543
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 12.199512481689453
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 12.182425498962402
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 12.192390441894531
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 12.184161186218262
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 12.196310043334961
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 12.20107364654541
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 12.187850952148438
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 12.196086883544922
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 12.170602798461914
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 12.188851356506348
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 12.183006286621094
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 12.201290130615234
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 12.201272010803223
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 12.194780349731445
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 12.184701919555664
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 12.158903121948242
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 12.186938285827637
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 12.186903953552246
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 12.204073905944824
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 12.193321228027344
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 12.194419860839844
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 12.200675964355469
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 12.172024726867676
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 12.184027671813965
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 12.187215805053711
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 12.200061798095703
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 12.199942588806152
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 12.194507598876953
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 12.194100379943848
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 12.16766357421875
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 12.170788764953613
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 12.181585311889648
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 12.196106910705566
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 12.201425552368164
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 12.198567390441895
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 12.1928129196167
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 12.175227165222168
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 12.192557334899902
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 12.189879417419434
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 12.200451850891113
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 12.19792652130127
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 12.19448471069336
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 12.190045356750488
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 12.183615684509277
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 12.18881607055664
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 12.189757347106934
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 12.198962211608887
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 12.197670936584473
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 12.194870948791504
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 12.201330184936523
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 12.187311172485352
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 12.189641952514648
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 12.189981460571289
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 12.197240829467773
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 12.197179794311523
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 12.18908405303955
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 12.17735481262207
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 12.18508243560791
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 12.19332504272461
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 12.19530200958252
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 12.206258773803711
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 12.201440811157227
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 12.199142456054688
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 12.19406795501709
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 12.192123413085938
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 12.196572303771973
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 12.186360359191895
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 12.196149826049805
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 12.196839332580566
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 12.19229507446289
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 12.200652122497559
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 12.170910835266113
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 12.187363624572754
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 12.190469741821289
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 12.197463989257812
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 12.201760292053223
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 12.199959754943848
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 12.195999145507812
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 12.195347785949707
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 12.187578201293945
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 12.196464538574219
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 12.196438789367676
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 12.197836875915527
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 12.197795867919922
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 12.203509330749512
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 12.191859245300293
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 12.186918258666992
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 12.19727611541748
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 12.19888687133789
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 12.199085235595703
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 12.201562881469727
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 12.187944412231445
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 12.192870140075684
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 12.1880464553833
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 12.190269470214844
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 12.19958209991455
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 12.198648452758789
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 12.194239616394043
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 12.202900886535645
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 12.194214820861816
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 12.19281005859375
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 12.194185256958008
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 12.198107719421387
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 12.194565773010254
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 12.197409629821777
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 12.191821098327637
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 12.188043594360352
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 12.19308853149414
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 12.191277503967285
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 12.202199935913086
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 12.19740104675293
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 12.199410438537598
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 12.193265914916992
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 12.182888984680176
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 12.184446334838867
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 12.191447257995605
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 12.198471069335938
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 12.198386192321777
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 12.196746826171875
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 12.194589614868164
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 12.189786911010742
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 12.195269584655762
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 12.188821792602539
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 12.1997652053833
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 12.20077133178711
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 12.19654369354248
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 12.205243110656738
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 12.189764022827148
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 12.187328338623047
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 12.18155288696289
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 12.195489883422852
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 12.196496963500977
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 12.197467803955078
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 12.194589614868164
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 12.187393188476562
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 12.185384750366211
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 12.179558753967285
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 12.196786880493164
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 12.196386337280273
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 12.182453155517578
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 12.204495429992676
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 12.181593894958496
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 12.182836532592773
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 12.180176734924316
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 12.195624351501465
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 12.198346138000488
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 12.191450119018555
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 12.200736045837402
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 12.184626579284668
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 12.184115409851074
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 12.17769718170166
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 12.197446823120117
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 12.198479652404785
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 12.186229705810547
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 12.203984260559082
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 12.183870315551758
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 12.184196472167969
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 12.179142951965332
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 12.197104454040527
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 12.201415061950684
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 12.188841819763184
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 12.210712432861328
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 12.178765296936035
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 12.165717124938965
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 12.177505493164062
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 12.19913101196289
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 12.197893142700195
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 12.197391510009766
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 12.202534675598145
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 12.150505065917969
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 12.1620512008667
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 12.1500244140625
+                }
+            ]
+        },
+        {
+            "bit": 5,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 11.051345825195312
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 11.050370216369629
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 11.043753623962402
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 11.050064086914062
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 11.048126220703125
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 11.046426773071289
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 11.049822807312012
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 11.050151824951172
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 11.051176071166992
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 11.04896068572998
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 11.059547424316406
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 11.046401023864746
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 11.048742294311523
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 11.049527168273926
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 11.051529884338379
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 11.05235767364502
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 11.048730850219727
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 11.04784107208252
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 11.042132377624512
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 11.052030563354492
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 11.045845031738281
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 11.051271438598633
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 11.049168586730957
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 11.053474426269531
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 11.048890113830566
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 11.044604301452637
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 11.052794456481934
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 11.047003746032715
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 11.050586700439453
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 11.049195289611816
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 11.048341751098633
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 11.051237106323242
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 11.042754173278809
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 11.0465669631958
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 11.051440238952637
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 11.051187515258789
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 11.051304817199707
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 11.051872253417969
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 11.050679206848145
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 11.039144515991211
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 11.04649829864502
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 11.045921325683594
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 11.053268432617188
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 11.052783966064453
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 11.052520751953125
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 11.050304412841797
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 11.043448448181152
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 11.048430442810059
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 11.047035217285156
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 11.052399635314941
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 11.05301284790039
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 11.048457145690918
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 11.04895305633545
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 11.040563583374023
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 11.050270080566406
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 11.047672271728516
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 11.053321838378906
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 11.051814079284668
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 11.050860404968262
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 11.051213264465332
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 11.04238224029541
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 11.046832084655762
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 11.04360580444336
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 11.051924705505371
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 11.052628517150879
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 11.051118850708008
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 11.050986289978027
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 11.047085762023926
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 11.05066204071045
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 11.049664497375488
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 11.051292419433594
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 11.050361633300781
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 11.05146598815918
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 11.051532745361328
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 11.04721450805664
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 11.049718856811523
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 11.051092147827148
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 11.051392555236816
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 11.049917221069336
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 11.051340103149414
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 11.048727035522461
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 11.045415878295898
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 11.04828929901123
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 11.049903869628906
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 11.054167747497559
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 11.052610397338867
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 11.052349090576172
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 11.051797866821289
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 11.05113410949707
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 11.050277709960938
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 11.053679466247559
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 11.050539016723633
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 11.051403045654297
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 11.051713943481445
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 11.046405792236328
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 11.043521881103516
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 11.0455322265625
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 11.050593376159668
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 11.054317474365234
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 11.050881385803223
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 11.052027702331543
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 11.053115844726562
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 11.050745010375977
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 11.047174453735352
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 11.050223350524902
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 11.050979614257812
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 11.048707962036133
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 11.054369926452637
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 11.045210838317871
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 11.04788875579834
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 11.050293922424316
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 11.051050186157227
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 11.051007270812988
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 11.050745010375977
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 11.050544738769531
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 11.052120208740234
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 11.05111312866211
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 11.052962303161621
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 11.05038070678711
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 11.052351951599121
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 11.0537748336792
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 11.0514554977417
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 11.052791595458984
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 11.050328254699707
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 11.050349235534668
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 11.049721717834473
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 11.051066398620605
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 11.051582336425781
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 11.05031681060791
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 11.051023483276367
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 11.049052238464355
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 11.043848991394043
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 11.048897743225098
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 11.053241729736328
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 11.051721572875977
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 11.053078651428223
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 11.053690910339355
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 11.047219276428223
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 11.046806335449219
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 11.045270919799805
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 11.05105209350586
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 11.052823066711426
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 11.051595687866211
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 11.047743797302246
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 11.049403190612793
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 11.050646781921387
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 11.047124862670898
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 11.051281929016113
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 11.050891876220703
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 11.050268173217773
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 11.053257942199707
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 11.046639442443848
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 11.050950050354004
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 11.050043106079102
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 11.050812721252441
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 11.048955917358398
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 11.050214767456055
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 11.049263954162598
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 11.043280601501465
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 11.048015594482422
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 11.049700736999512
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 11.050586700439453
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 11.049943923950195
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 11.049458503723145
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 11.052498817443848
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 11.046576499938965
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 11.048099517822266
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 11.042832374572754
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 11.051060676574707
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 11.051395416259766
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 11.05078125
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 11.052319526672363
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 11.04311752319336
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 11.045637130737305
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 11.043219566345215
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 11.05045223236084
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 11.0486421585083
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 11.052265167236328
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 11.049090385437012
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 11.043232917785645
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 11.044672966003418
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 11.043222427368164
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 11.05155086517334
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 11.050307273864746
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 11.049322128295898
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 11.054634094238281
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 11.042032241821289
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 11.047428131103516
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 11.046034812927246
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 11.051033973693848
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 11.050786972045898
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 11.049223899841309
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 11.057891845703125
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 11.026429176330566
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 11.032037734985352
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 11.031327247619629
+                }
+            ]
+        },
+        {
+            "bit": 6,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.676329612731934
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.673662185668945
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.67713451385498
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.675612449645996
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.675748825073242
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.674802780151367
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.674924850463867
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.676283836364746
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.676477432250977
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.675711631774902
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.67593765258789
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.672209739685059
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.676039695739746
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.673112869262695
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.676030158996582
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.676111221313477
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.676067352294922
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.67630386352539
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.672319412231445
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.672661781311035
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.673912048339844
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.676401138305664
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.67438793182373
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.676172256469727
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.675345420837402
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.671924591064453
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.675069808959961
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.67390251159668
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.676050186157227
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.677979469299316
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.676701545715332
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.6752347946167
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.67084789276123
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.672741889953613
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.674308776855469
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.676968574523926
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.676559448242188
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.675153732299805
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.675271034240723
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.672761917114258
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.67199420928955
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.676680564880371
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.6766939163208
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.676372528076172
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.6771240234375
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.6764497756958
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.672995567321777
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.676703453063965
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.674988746643066
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.675760269165039
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.677274703979492
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.677233695983887
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.675841331481934
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.671433448791504
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.67463493347168
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.6738862991333
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.677090644836426
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.676355361938477
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.67544937133789
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.678918838500977
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.670843124389648
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.673418045043945
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.675326347351074
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.67662525177002
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.677327156066895
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.674955368041992
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.677999496459961
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.676283836364746
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.674386024475098
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.67389965057373
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.677291870117188
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.676925659179688
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.677375793457031
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.67282485961914
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.675745010375977
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.67618465423584
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.676199913024902
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.67612361907959
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.675713539123535
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.675762176513672
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.674912452697754
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.675148963928223
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.674260139465332
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.67700481414795
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.676989555358887
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.677145004272461
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.675164222717285
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.677057266235352
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.674854278564453
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.676403999328613
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.673762321472168
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.676790237426758
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.676335334777832
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.676385879516602
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.675676345825195
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.671753883361816
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.672537803649902
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.67525577545166
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.67575740814209
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.67612361907959
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.676294326782227
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.677987098693848
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.676390647888184
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.675734519958496
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.676436424255371
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.676431655883789
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.676065444946289
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.676050186157227
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.677923202514648
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.676238059997559
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.675326347351074
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.673474311828613
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.676054954528809
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.676846504211426
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.677567481994629
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.67350959777832
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.674263000488281
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.674751281738281
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.676502227783203
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.675909996032715
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.676013946533203
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.677160263061523
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.676111221313477
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.675052642822266
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.67632007598877
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.676314353942871
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.676329612731934
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.676634788513184
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.677085876464844
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.676518440246582
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.674410820007324
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.67582893371582
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.67530345916748
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.677226066589355
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.674527168273926
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.677281379699707
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.676190376281738
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.67458438873291
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.676701545715332
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.674578666687012
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.677021980285645
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.675920486450195
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.676774978637695
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.675968170166016
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.675092697143555
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.676567077636719
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.675466537475586
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.676915168762207
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.676345825195312
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.675810813903809
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.680309295654297
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.6746244430542
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.675800323486328
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.673392295837402
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.676492691040039
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.67574691772461
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.67611312866211
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.676393508911133
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.67338752746582
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.675232887268066
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.672069549560547
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.676436424255371
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.676228523254395
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.675576210021973
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.678200721740723
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.672150611877441
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.673097610473633
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.670710563659668
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.677223205566406
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.675731658935547
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.675774574279785
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.676533699035645
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.672196388244629
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.675130844116211
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.672260284423828
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.676352500915527
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.67531681060791
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.675251007080078
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.67506217956543
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.672934532165527
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.672183990478516
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.67196273803711
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.677040100097656
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.67768383026123
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.675640106201172
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.679840087890625
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.673463821411133
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.669708251953125
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.671504974365234
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.676878929138184
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.676836013793945
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.675243377685547
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.677521705627441
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.660104751586914
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.666590690612793
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.663055419921875
+                }
+            ]
+        },
+        {
+            "bit": 7,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.442667961120605
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.444175720214844
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.443778038024902
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.440760612487793
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.44139289855957
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.442167282104492
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.441547393798828
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.4439697265625
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.443235397338867
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.44385814666748
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.443076133728027
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.443568229675293
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.44163703918457
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.442699432373047
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.443207740783691
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.443011283874512
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.442501068115234
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.44402027130127
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.441267967224121
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.444022178649902
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.440688133239746
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.443564414978027
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.442070007324219
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.44461441040039
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.441737174987793
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.441076278686523
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.444948196411133
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.441740989685059
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.443011283874512
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.442789077758789
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.441352844238281
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.441828727722168
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.44018268585205
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.442389488220215
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.443504333496094
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.444395065307617
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.443867683410645
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.443160057067871
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.443187713623047
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.438167572021484
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.441858291625977
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.440536499023438
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.443629264831543
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.443604469299316
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.442819595336914
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.442479133605957
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.441357612609863
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.441352844238281
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.440287590026855
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.444184303283691
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.44356632232666
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.442991256713867
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.444071769714355
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.440995216369629
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.442502975463867
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.444186210632324
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.443278312683105
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.443392753601074
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.444107055664062
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.440227508544922
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.44158935546875
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.444012641906738
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.442598342895508
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.444076538085938
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.443713188171387
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.443499565124512
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.442349433898926
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.441582679748535
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.443160057067871
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.443716049194336
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.442867279052734
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.443025588989258
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.443700790405273
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.446552276611328
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.442344665527344
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.4426851272583
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.44405460357666
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.443264961242676
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.443314552307129
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.443138122558594
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.441915512084961
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.442842483520508
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.442408561706543
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.442349433898926
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.44381046295166
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.443429946899414
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.44421672821045
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.44393539428711
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.443354606628418
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.443439483642578
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.444048881530762
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.442757606506348
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.443404197692871
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.442607879638672
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.441681861877441
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.441627502441406
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.442399024963379
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.442588806152344
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.443036079406738
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.442946434020996
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.444198608398438
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.44216251373291
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.443787574768066
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.443464279174805
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.445137977600098
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.443564414978027
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.442192077636719
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.44461441040039
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.443306922912598
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.443967819213867
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.442705154418945
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.442976951599121
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.443459510803223
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.443655967712402
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.444535255432129
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.445446968078613
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.443489074707031
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.443005561828613
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.443591117858887
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.443283081054688
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.443267822265625
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.444465637207031
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.443890571594238
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.444104194641113
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.442665100097656
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.442876815795898
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.443671226501465
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.443573951721191
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.443914413452148
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.444151878356934
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.443058967590332
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.442760467529297
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.442747116088867
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.444037437438965
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.443416595458984
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.442771911621094
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.443593978881836
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.442296981811523
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.442480087280273
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.443371772766113
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.443604469299316
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.444074630737305
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.443434715270996
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.442802429199219
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.443357467651367
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.443416595458984
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.440210342407227
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.444085121154785
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.443803787231445
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.443076133728027
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.443132400512695
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.44211196899414
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.442218780517578
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.443513870239258
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.443655967712402
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.443093299865723
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.443258285522461
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.442742347717285
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.44183349609375
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.440402030944824
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.44167423248291
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.4431734085083
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.442428588867188
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.443385124206543
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.443155288696289
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.442461013793945
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.442896842956543
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.44083309173584
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.443609237670898
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.443379402160645
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.442662239074707
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.443798065185547
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.442328453063965
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.44071102142334
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.441168785095215
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.443349838256836
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.444764137268066
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.442145347595215
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.444584846496582
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.440247535705566
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.441627502441406
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.438939094543457
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.443320274353027
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.443357467651367
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.443387031555176
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.444435119628906
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.440255165100098
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.441023826599121
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.44118595123291
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.443329811096191
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.44332504272461
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.44328498840332
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.445371627807617
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.433932304382324
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.433636665344238
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.436285972595215
+                }
+            ]
+        },
+        {
+            "bit": 8,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.317365646362305
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.3170804977417
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.315113067626953
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.316485404968262
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.317566871643066
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.317193984985352
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.316753387451172
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.317429542541504
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.31772232055664
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.317630767822266
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.317286491394043
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.315831184387207
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.317183494567871
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.317538261413574
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.317666053771973
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.319061279296875
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.317885398864746
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.316733360290527
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.315624237060547
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.317848205566406
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.318243980407715
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.317793846130371
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.316180229187012
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.317112922668457
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.319324493408203
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.317779541015625
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.316474914550781
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.31709098815918
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.317477226257324
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.318207740783691
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.317533493041992
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.31690788269043
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.317292213439941
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.318745613098145
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.31567668914795
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.317708015441895
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.318659782409668
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.317055702209473
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.316893577575684
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.316622734069824
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.316563606262207
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.319725036621094
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.31889820098877
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.318089485168457
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.3182954788208
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.31832504272461
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.315890312194824
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.31530475616455
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.318611145019531
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.317521095275879
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.31735610961914
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.317222595214844
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.315566062927246
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.315840721130371
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.317646980285645
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.316062927246094
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.317882537841797
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.31795597076416
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.316957473754883
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.318022727966309
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.314935684204102
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.315262794494629
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.315518379211426
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.317658424377441
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.317502975463867
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.317813873291016
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.317164421081543
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.317800521850586
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.316060066223145
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.316571235656738
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.318058967590332
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.317429542541504
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.318258285522461
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.314087867736816
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.315796852111816
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.317230224609375
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.316328048706055
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.31758975982666
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.318217277526855
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.31786060333252
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.317331314086914
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.316967010498047
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.316024780273438
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.315730094909668
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.31806468963623
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.318138122558594
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.318977355957031
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.316494941711426
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.317304611206055
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.316577911376953
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.317169189453125
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.31808090209961
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.31832218170166
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.318307876586914
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.316746711730957
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.315695762634277
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.316283226013184
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.315719604492188
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.318147659301758
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.317513465881348
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.317705154418945
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.320101737976074
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.319439888000488
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.317719459533691
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.318055152893066
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.317449569702148
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.317341804504395
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.316362380981445
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.317253112792969
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.31749439239502
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.316229820251465
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.31772518157959
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.317927360534668
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.317971229553223
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.318187713623047
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.31570053100586
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.317341804504395
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.316352844238281
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.317414283752441
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.318187713623047
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.318153381347656
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.318192481994629
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.316490173339844
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.317496299743652
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.31812572479248
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.317825317382812
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.318002700805664
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.317459106445312
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.31836223602295
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.316244125366211
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.317474365234375
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.317144393920898
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.317329406738281
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.318182945251465
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.31694507598877
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.318408966064453
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.318197250366211
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.316658020019531
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.315673828125
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.31677532196045
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.318105697631836
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.31755542755127
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.318039894104004
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.317002296447754
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.316544532775879
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.316225051879883
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.317831039428711
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.317835807800293
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.317527770996094
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.318079948425293
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.31860065460205
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.31701374053955
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.317764282226562
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.316150665283203
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.317577362060547
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.317710876464844
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.317758560180664
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.317675590515137
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.317619323730469
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.318448066711426
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.315346717834473
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.317459106445312
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.318155288696289
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.317371368408203
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.31914234161377
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.315624237060547
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.315651893615723
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.315954208374023
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.317832946777344
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.317158699035645
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.318055152893066
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.318005561828613
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.316258430480957
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.31711483001709
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.31611156463623
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.317672729492188
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.317630767822266
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.317276954650879
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.316947937011719
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.315230369567871
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.31496524810791
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.314921379089355
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.317538261413574
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.317633628845215
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.31742000579834
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.319570541381836
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.316052436828613
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.316691398620605
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.31684398651123
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.317815780639648
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.317774772644043
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.317429542541504
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.31742000579834
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.310721397399902
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.312508583068848
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.312742233276367
+                }
+            ]
+        },
+        {
+            "bit": 9,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.213951110839844
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.213761329650879
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.216739654541016
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.211606979370117
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.21341323852539
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.213969230651855
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.213164329528809
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.214395523071289
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.213720321655273
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.213849067687988
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.213082313537598
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.214737892150879
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.214932441711426
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.212729454040527
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.214231491088867
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.21406078338623
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.214421272277832
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.213342666625977
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.213257789611816
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.213922500610352
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.21264362335205
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.214348793029785
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.213433265686035
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.21391487121582
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.213057518005371
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.211932182312012
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.214153289794922
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.21359634399414
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.213364601135254
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.214436531066895
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.213072776794434
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.213208198547363
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.212361335754395
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.212300300598145
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.214153289794922
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.214387893676758
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.213603019714355
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.214983940124512
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.213890075683594
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.212504386901855
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.213471412658691
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.209765434265137
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.213452339172363
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.213817596435547
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.214051246643066
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.213228225708008
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.213744163513184
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.214043617248535
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.211981773376465
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.214202880859375
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.21406078338623
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.213933944702148
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.214494705200195
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.212721824645996
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.212726593017578
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.213179588317871
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.21370792388916
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.214151382446289
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.214792251586914
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.213155746459961
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.213457107543945
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.214986801147461
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.214543342590332
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.214367866516113
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.21358871459961
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.21347713470459
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.21450138092041
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.212875366210938
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.214035987854004
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.21346664428711
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.213812828063965
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.213783264160156
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.213568687438965
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.21700382232666
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.213177680969238
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.213257789611816
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.215493202209473
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.213749885559082
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.214140892028809
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.214338302612305
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.213455200195312
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.213521003723145
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.213350296020508
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.213571548461914
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.214217185974121
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.214635848999023
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.21362018585205
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.21429443359375
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.213642120361328
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.213496208190918
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.21414852142334
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.213725090026855
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.214071273803711
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.213593482971191
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.212994575500488
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.213433265686035
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.21341323852539
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.213905334472656
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.214151382446289
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.213715553283691
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.214115142822266
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.212843894958496
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.21362018585205
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.212544441223145
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.213496208190918
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.213642120361328
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.214140892028809
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.215787887573242
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.21568775177002
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.214309692382812
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.214046478271484
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.21401309967041
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.214167594909668
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.213822364807129
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.214178085327148
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.216797828674316
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.21412181854248
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.213715553283691
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.21450424194336
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.213977813720703
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.213690757751465
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.214344024658203
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.21536636352539
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.21386432647705
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.213590621948242
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.213690757751465
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.213715553283691
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.214057922363281
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.21364688873291
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.215076446533203
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.214057922363281
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.214007377624512
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.214311599731445
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.214374542236328
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.213846206665039
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.214651107788086
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.213963508605957
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.213586807250977
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.213808059692383
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.214095115661621
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.213929176330566
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.213839530944824
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.213808059692383
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.213933944702148
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.213846206665039
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.214178085327148
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.212376594543457
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.214373588562012
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.214221954345703
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.214051246643066
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.214217185974121
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.213802337646484
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.213491439819336
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.213583946228027
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.214076042175293
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.214016914367676
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.213706016540527
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.213676452636719
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.2133207321167
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.21264934539795
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.213094711303711
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.214162826538086
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.213802337646484
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.214005470275879
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.214118957519531
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.213593482971191
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.21373462677002
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.212762832641602
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.214117050170898
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.21435832977295
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.213485717773438
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.213630676269531
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.21264934539795
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.213554382324219
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.212656021118164
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.213910102844238
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.214126586914062
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.21385669708252
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.214200019836426
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.21375846862793
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.21373462677002
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.21338176727295
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.21374225616455
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.213827133178711
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.213359832763672
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.214611053466797
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.212215423583984
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.2135591506958
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.212570190429688
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.213678359985352
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.213868141174316
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.213793754577637
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.214316368103027
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.208498001098633
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.209223747253418
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.209026336669922
+                }
+            ]
+        },
+        {
+            "bit": 10,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.159443855285645
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.159516334533691
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.159503936767578
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.160867691040039
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.160138130187988
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.159215927124023
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.159904479980469
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.159976959228516
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.160082817077637
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.160306930541992
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.159223556518555
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.15882396697998
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.159478187561035
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.158843040466309
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.159744262695312
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.15987491607666
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.159734725952148
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.158673286437988
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.158978462219238
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.16012191772461
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.160518646240234
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.16006851196289
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.159574508666992
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.160447120666504
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.15939998626709
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.159942626953125
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.160526275634766
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.159462928771973
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.15990161895752
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.159981727600098
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.159482955932617
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.159584999084473
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.159601211547852
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.160233497619629
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.159700393676758
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.160030364990234
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.16003704071045
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.160165786743164
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.159996032714844
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.159584999084473
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.160181999206543
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.163278579711914
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.160126686096191
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.160181999206543
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.16018009185791
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.160543441772461
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.157787322998047
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.159002304077148
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.160529136657715
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.159964561462402
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.16012954711914
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.16020393371582
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.159725189208984
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.158232688903809
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.16085433959961
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.159884452819824
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.160082817077637
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.160285949707031
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.159063339233398
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.160917282104492
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.158402442932129
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.159138679504395
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.158926010131836
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.159860610961914
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.160266876220703
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.159937858581543
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.159823417663574
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.159814834594727
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.159193992614746
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.161362648010254
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.160005569458008
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.159782409667969
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.160364151000977
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.15740966796875
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.160640716552734
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.159710884094238
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.159346580505371
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.159823417663574
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.16012191772461
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.159560203552246
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.160306930541992
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.160096168518066
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.159133911132812
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.161136627197266
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.160429954528809
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.16012191772461
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.16037368774414
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.159971237182617
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.160025596618652
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.161294937133789
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.157772064208984
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.160117149353027
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.16015625
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.16006088256836
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.158160209655762
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.158331871032715
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.159371376037598
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.159148216247559
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.160107612609863
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.160348892211914
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.16049575805664
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.160524368286133
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.160826683044434
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.159915924072266
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.16154956817627
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.159865379333496
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.159821510314941
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.16028881072998
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.157276153564453
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.159993171691895
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.159945487976074
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.159801483154297
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.15976619720459
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.159865379333496
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.160097122192383
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.15900993347168
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.160225868225098
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.159300804138184
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.159690856933594
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.160011291503906
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.160244941711426
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.160192489624023
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.159380912780762
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.159879684448242
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.158867835998535
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.160022735595703
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.160131454467773
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.159662246704102
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.160558700561523
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.159102439880371
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.160075187683105
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.159753799438477
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.15918254852295
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.159967422485352
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.160307884216309
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.159797668457031
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.160555839538574
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.160082817077637
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.160332679748535
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.159337043762207
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.160126686096191
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.160258293151855
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.160223007202148
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.159675598144531
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.159651756286621
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.159618377685547
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.160615921020508
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.16015625
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.159845352172852
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.160097122192383
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.160592079162598
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.159804344177246
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.160558700561523
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.159385681152344
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.159778594970703
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.160005569458008
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.159843444824219
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.159841537475586
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.15971565246582
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.160238265991211
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.159189224243164
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.160063743591309
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.159926414489746
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.159725189208984
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.159380912780762
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.159157752990723
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.159201622009277
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.158857345581055
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.160000801086426
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.159693717956543
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.160151481628418
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.160019874572754
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.159177780151367
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.158770561218262
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.158663749694824
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.160238265991211
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.159729957580566
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.159245491027832
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.159934997558594
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.15785026550293
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.15863037109375
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.15773868560791
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.160019874572754
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.160088539123535
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.15979290008545
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.160773277282715
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.159184455871582
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.158855438232422
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.159404754638672
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.160126686096191
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.15981674194336
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.159521102905273
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.160529136657715
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.156387329101562
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.156983375549316
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.158339500427246
+                }
+            ]
+        },
+        {
+            "bit": 11,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.114678382873535
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.11498737335205
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.11557388305664
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.113097190856934
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.115793228149414
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.11535930633545
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.11493968963623
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.115370750427246
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.115321159362793
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.114891052246094
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.114230155944824
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.115670204162598
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.115952491760254
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.115057945251465
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.11525821685791
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.115119934082031
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.115768432617188
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.115797996520996
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.115706443786621
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.114888191223145
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.114531517028809
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.115021705627441
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.114688873291016
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.115242958068848
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.115720748901367
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.115070343017578
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.115602493286133
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.114669799804688
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.114949226379395
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.115696907043457
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.115137100219727
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.114550590515137
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.11379337310791
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.115957260131836
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.116581916809082
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.11525821685791
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.115097999572754
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.115547180175781
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.114818572998047
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.11240291595459
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.113762855529785
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.111863136291504
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.115118026733398
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.115209579467773
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.115580558776855
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.115226745605469
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.115239143371582
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.115348815917969
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.114912986755371
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.115245819091797
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.115004539489746
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.115236282348633
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.115070343017578
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.11464786529541
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.114225387573242
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.114992141723633
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.115100860595703
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.11535930633545
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.116206169128418
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.11377239227295
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.115071296691895
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.11562442779541
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.115446090698242
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.115653038024902
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.11485767364502
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.11501693725586
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.115130424499512
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.114977836608887
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.115002632141113
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.113746643066406
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.115070343017578
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.11530590057373
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.115286827087402
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.115774154663086
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.11483383178711
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.1151762008667
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.115108489990234
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.11543083190918
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.115111351013184
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.115422248840332
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.113521575927734
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.114842414855957
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.114678382873535
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.115253448486328
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.11543083190918
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.115411758422852
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.114958763122559
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.114283561706543
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.115660667419434
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.114770889282227
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.116944313049316
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.115198135375977
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.115242958068848
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.114842414855957
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.115463256835938
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.114239692687988
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.113398551940918
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.114567756652832
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.115419387817383
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.115286827087402
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.115057945251465
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.1145658493042
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.11538314819336
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.116000175476074
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.114861488342285
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.114891052246094
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.11485767364502
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.115585327148438
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.115937232971191
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.115071296691895
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.11401081085205
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.115448951721191
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.115591049194336
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.115580558776855
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.115364074707031
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.116422653198242
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.116277694702148
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.115411758422852
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.115288734436035
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.115476608276367
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.115226745605469
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.115452766418457
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.116097450256348
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.115653038024902
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.115310668945312
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.114269256591797
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.115127563476562
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.115217208862305
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.114628791809082
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.115281105041504
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.115171432495117
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.114706039428711
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.115909576416016
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.115368843078613
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.11512279510498
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.115850448608398
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.115561485290527
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.115137100219727
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.11483097076416
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.115103721618652
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.115351676940918
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.115159034729004
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.115460395812988
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.115528106689453
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.115113258361816
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.115198135375977
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.113919258117676
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.11522102355957
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.114992141723633
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.115078926086426
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.114684104919434
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.115280151367188
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.114272117614746
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.114958763122559
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.11522388458252
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.115026473999023
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.1151762008667
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.115498542785645
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.115084648132324
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.11492919921875
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.115078926086426
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.115537643432617
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.11546802520752
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.115233421325684
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.115509033203125
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.115002632141113
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.11538314819336
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.114534378051758
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.11523151397705
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.11532974243164
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.114760398864746
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.114768028259277
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.114775657653809
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.115050315856934
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.114840507507324
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.115021705627441
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.115334510803223
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.115446090698242
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.115240097045898
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.114949226379395
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.114744186401367
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.114659309387207
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.115050315856934
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.11498737335205
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.115078926086426
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.115821838378906
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.114676475524902
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.114129066467285
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.113984107971191
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.11535930633545
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.114893913269043
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.115152359008789
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.115740776062012
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.110855102539062
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.110512733459473
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.111279487609863
+                }
+            ]
+        },
+        {
+            "bit": 12,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.074488639831543
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.074613571166992
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.074769973754883
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.074707984924316
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.074085235595703
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.074605941772461
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.075084686279297
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.074618339538574
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.075101852416992
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.074952125549316
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.07343864440918
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.073912620544434
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.07400131225586
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.073982238769531
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.074686050415039
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.075163841247559
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.074203491210938
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.073705673217773
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.073734283447266
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.075350761413574
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.076225280761719
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.074909210205078
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.073895454406738
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.074774742126465
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.074624061584473
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.074627876281738
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.074434280395508
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.074524879455566
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.074461936950684
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.074477195739746
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.074207305908203
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.073734283447266
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.074207305908203
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.073333740234375
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.073847770690918
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.07474136352539
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.074817657470703
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.074397087097168
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.074431419372559
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.075535774230957
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.073775291442871
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.076271057128906
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.074442863464355
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.074882507324219
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.074332237243652
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.074479103088379
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.073107719421387
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.073724746704102
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.074385643005371
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.0750150680542
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.07443618774414
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.074814796447754
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.074029922485352
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.073472023010254
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.075451850891113
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.074409484863281
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.074918746948242
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.074912071228027
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.074328422546387
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.075329780578613
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.073533058166504
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.073862075805664
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.073436737060547
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.074251174926758
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.074806213378906
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.074466705322266
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.074798583984375
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.074116706848145
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.073967933654785
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.076528549194336
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.074931144714355
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.074645042419434
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.074697494506836
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.073182106018066
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.074164390563965
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.074132919311523
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.074769973754883
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.074589729309082
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.074461936950684
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.074188232421875
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.075404167175293
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.074978828430176
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.074481010437012
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.074746131896973
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.075056076049805
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.074822425842285
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.074813842773438
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.075576782226562
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.074399948120117
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.074385643005371
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.072660446166992
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.07463550567627
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.074543952941895
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.074395179748535
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.073175430297852
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.074841499328613
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.074836730957031
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.0737943649292
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.07450008392334
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.074673652648926
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.07524585723877
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.075370788574219
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.074860572814941
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.073281288146973
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.075807571411133
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.074567794799805
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.074453353881836
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.074995994567871
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.074630737304688
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.075178146362305
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.074337005615234
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.073760986328125
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.074426651000977
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.074906349182129
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.074803352355957
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.074399948120117
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.074188232421875
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.07400131225586
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.074145317077637
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.07474136352539
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.07474136352539
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.074974060058594
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.074039459228516
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.075096130371094
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.074399948120117
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.074719429016113
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.074976921081543
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.074605941772461
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.074798583984375
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.07425594329834
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.074901580810547
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.074702262878418
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.073847770690918
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.074711799621582
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.074716567993164
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.074352264404297
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.074539184570312
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.074317932128906
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.074515342712402
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.074524879455566
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.074841499328613
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.074649810791016
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.074959754943848
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.074846267700195
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.07473373413086
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.074515342712402
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.074700355529785
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.074957847595215
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.074771881103516
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.07443618774414
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.075356483459473
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.074092864990234
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.075118064880371
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.074090003967285
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.074645042419434
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.074481010437012
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.074610710144043
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.074455261230469
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.074322700500488
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.07502269744873
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.07427978515625
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.074668884277344
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.074458122253418
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.07486629486084
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.074268341064453
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.074399948120117
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.073786735534668
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.07357406616211
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.074649810791016
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.074621200561523
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.0746431350708
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.074904441833496
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.074735641479492
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.07352066040039
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.07412338256836
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.074678421020508
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.074735641479492
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.074515342712402
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.07473373413086
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.073994636535645
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.073631286621094
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.072929382324219
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.074707984924316
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.074707984924316
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.074597358703613
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.0748929977417
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.073929786682129
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.074572563171387
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.074678421020508
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.074390411376953
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.074613571166992
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.074548721313477
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.074966430664062
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.072348594665527
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.073851585388184
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.073240280151367
+                }
+            ]
+        },
+        {
+            "bit": 13,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.037837028503418
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.037497520446777
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.038172721862793
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.037765502929688
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.038607597351074
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.037389755249023
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.03767204284668
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.037751197814941
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.037851333618164
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.03796672821045
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.035980224609375
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.038126945495605
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.038559913635254
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.038686752319336
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.037540435791016
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.037736892700195
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.037796974182129
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.03801441192627
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.037736892700195
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.037981033325195
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.037425994873047
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.038307189941406
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.037239074707031
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.037713050842285
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.037755966186523
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.037473678588867
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.03882884979248
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.038784980773926
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.037693977355957
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.038105010986328
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.038002014160156
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.0375337600708
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.037487983703613
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.038326263427734
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.038483619689941
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.038315773010254
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.037832260131836
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.038098335266113
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.037765502929688
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.03662109375
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.037353515625
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.036392211914062
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.037985801696777
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.038077354431152
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.03810977935791
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.038043022155762
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.03737735748291
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.038421630859375
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.036918640136719
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.037894248962402
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.037885665893555
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.037453651428223
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.03769588470459
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.037691116333008
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.036685943603516
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.037851333618164
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.038227081298828
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.03782844543457
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.037947654724121
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.037471771240234
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.037161827087402
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.037511825561523
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.038047790527344
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.038115501403809
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.037602424621582
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.038165092468262
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.037291526794434
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.036972999572754
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.038637161254883
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.036497116088867
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.038139343261719
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.037920951843262
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.03791332244873
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.038204193115234
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.037674903869629
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.0376558303833
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.037878036499023
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.038162231445312
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.038026809692383
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.038066864013672
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.037220001220703
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.038277626037598
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.0366792678833
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.037758827209473
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.037801742553711
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.03806209564209
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.037935256958008
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.037487983703613
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.037296295166016
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.037334442138672
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.038459777832031
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.038066864013672
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.037995338439941
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.03797435760498
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.03823184967041
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.03723430633545
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.037243843078613
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.038268089294434
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.03774642944336
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.0376558303833
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.037732124328613
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.03738021850586
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.038191795349121
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.039052963256836
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.037981033325195
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.037724494934082
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.038239479064941
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.037848472595215
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.038201332092285
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.03791618347168
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.037928581237793
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.03803825378418
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.037982940673828
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.037593841552734
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.037954330444336
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.038829803466797
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.038223266601562
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.03808879852295
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.038559913635254
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.037693977355957
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.03789234161377
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.038385391235352
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.038268089294434
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.037931442260742
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.037464141845703
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.037829399108887
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.0376558303833
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.038033485412598
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.037866592407227
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.038328170776367
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.037617683410645
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.037418365478516
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.03862190246582
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.03793716430664
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.037602424621582
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.038540840148926
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.038153648376465
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.038212776184082
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.036703109741211
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.037593841552734
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.038077354431152
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.037841796875
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.03828239440918
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.038077354431152
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.03791618347168
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.03791618347168
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.03750228881836
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.037823677062988
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.037724494934082
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.037879943847656
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.037856101989746
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.038250923156738
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.037193298339844
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.038023948669434
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.038115501403809
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.037931442260742
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.037760734558105
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.037636756896973
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.038153648376465
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.037952423095703
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.037487983703613
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.037969589233398
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.03782844543457
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.037689208984375
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.038021087646484
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.037702560424805
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.037736892700195
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.037717819213867
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.037848472595215
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.037813186645508
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.037806510925293
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.037440299987793
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.036966323852539
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.038594245910645
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.036928176879883
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.03799057006836
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.037674903869629
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.037705421447754
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.037717819213867
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.037348747253418
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.03812026977539
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.037789344787598
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.037617683410645
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.037808418273926
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.037875175476074
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.038077354431152
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.036784172058105
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.037372589111328
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.037145614624023
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.038139343261719
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.03786849975586
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.037640571594238
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.038837432861328
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.035650253295898
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.034812927246094
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.03567886352539
+                }
+            ]
+        },
+        {
+            "bit": 14,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 10.015263557434082
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 10.015190124511719
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 10.015426635742188
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 10.015303611755371
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 10.014955520629883
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 10.015273094177246
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 10.0151948928833
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 10.01530647277832
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 10.015146255493164
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 10.015743255615234
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 10.01552677154541
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 10.014817237854004
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 10.014325141906738
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 10.015327453613281
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 10.015433311462402
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 10.01538372039795
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 10.015168190002441
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 10.01517105102539
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 10.01501750946045
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 10.014979362487793
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 10.015968322753906
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 10.015165328979492
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 10.014626502990723
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 10.015384674072266
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 10.015079498291016
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 10.015252113342285
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 10.014726638793945
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 10.01480484008789
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 10.014936447143555
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 10.015020370483398
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 10.01506519317627
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 10.014545440673828
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 10.015233039855957
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 10.014796257019043
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 10.014850616455078
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 10.01578140258789
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 10.01483154296875
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 10.015399932861328
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 10.015314102172852
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 10.016201972961426
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 10.01551628112793
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 10.015071868896484
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 10.015182495117188
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 10.015022277832031
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 10.015244483947754
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 10.015098571777344
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 10.014955520629883
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 10.014191627502441
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 10.014907836914062
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 10.01492691040039
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 10.015399932861328
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 10.015318870544434
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 10.015079498291016
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 10.014678001403809
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 10.016549110412598
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 10.01490306854248
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 10.014941215515137
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 10.01518726348877
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 10.01502513885498
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 10.015098571777344
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 10.01455020904541
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 10.015690803527832
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 10.014812469482422
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 10.015233039855957
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 10.01487922668457
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 10.015101432800293
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 10.016185760498047
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 10.015519142150879
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 10.01453971862793
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 10.016302108764648
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 10.015203475952148
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 10.015146255493164
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 10.015483856201172
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 10.014901161193848
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 10.015103340148926
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 10.015252113342285
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 10.01489543914795
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 10.014793395996094
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 10.015446662902832
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 10.015361785888672
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 10.015376091003418
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 10.015089988708496
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 10.014914512634277
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 10.015810012817383
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 10.015342712402344
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 10.014967918395996
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 10.015339851379395
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 10.015655517578125
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 10.01516342163086
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 10.015337944030762
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 10.014488220214844
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 10.015151977539062
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 10.01504135131836
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 10.014688491821289
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 10.01480770111084
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 10.01489543914795
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 10.015165328979492
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 10.01407241821289
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 10.015376091003418
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 10.015079498291016
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 10.015519142150879
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 10.015989303588867
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 10.01566219329834
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 10.014296531677246
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 10.015528678894043
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 10.015342712402344
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 10.014984130859375
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 10.015480995178223
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 10.014979362487793
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 10.015727043151855
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 10.014683723449707
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 10.014626502990723
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 10.015314102172852
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 10.015179634094238
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 10.015485763549805
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 10.014742851257324
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 10.015292167663574
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 10.01483154296875
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 10.014697074890137
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 10.01506519317627
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 10.01518726348877
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 10.015337944030762
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 10.015103340148926
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 10.015155792236328
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 10.014817237854004
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 10.014955520629883
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 10.015514373779297
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 10.015237808227539
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 10.015007972717285
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 10.014554023742676
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 10.015588760375977
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 10.01490306854248
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 10.014657020568848
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 10.015342712402344
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 10.015325546264648
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 10.014564514160156
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 10.015411376953125
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 10.01488208770752
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 10.015027046203613
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 10.014534950256348
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 10.015244483947754
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 10.015299797058105
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 10.015571594238281
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 10.014925003051758
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 10.01489543914795
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 10.014577865600586
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 10.014604568481445
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 10.015146255493164
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 10.015155792236328
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 10.015134811401367
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 10.01516056060791
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 10.015271186828613
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 10.015349388122559
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 10.014480590820312
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 10.014798164367676
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 10.01480484008789
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 10.01538372039795
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 10.015230178833008
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 10.014912605285645
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 10.0148286819458
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 10.01502513885498
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 10.01499080657959
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 10.015244483947754
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 10.015260696411133
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 10.015103340148926
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 10.014914512634277
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 10.014410972595215
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 10.01432991027832
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 10.015314102172852
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 10.014971733093262
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 10.01501750946045
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 10.015220642089844
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 10.015168190002441
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 10.014021873474121
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 10.01499080657959
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 10.014782905578613
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 10.0155668258667
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 10.015299797058105
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 10.014809608459473
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 10.01516342163086
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 10.014196395874023
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 10.014545440673828
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 10.01534652709961
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 10.01504135131836
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 10.015314102172852
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 10.015058517456055
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 10.015222549438477
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 10.015060424804688
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 10.014955520629883
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 10.015084266662598
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 10.015395164489746
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 10.015146255493164
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 10.015643119812012
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 10.013763427734375
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 10.013914108276367
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 10.013551712036133
+                }
+            ]
+        },
+        {
+            "bit": 15,
+            "ppls": [
+                {
+                    "layer": "layer.0.self_attn.q_proj",
+                    "ppl": 9.991853713989258
+                },
+                {
+                    "layer": "layer.0.self_attn.k_proj",
+                    "ppl": 9.991497039794922
+                },
+                {
+                    "layer": "layer.0.self_attn.v_proj",
+                    "ppl": 9.991786003112793
+                },
+                {
+                    "layer": "layer.0.self_attn.o_proj",
+                    "ppl": 9.992063522338867
+                },
+                {
+                    "layer": "layer.0.mlp.gate_proj",
+                    "ppl": 9.991840362548828
+                },
+                {
+                    "layer": "layer.0.mlp.up_proj",
+                    "ppl": 9.992063522338867
+                },
+                {
+                    "layer": "layer.0.mlp.down_proj",
+                    "ppl": 9.992478370666504
+                },
+                {
+                    "layer": "layer.1.self_attn.q_proj",
+                    "ppl": 9.991342544555664
+                },
+                {
+                    "layer": "layer.1.self_attn.k_proj",
+                    "ppl": 9.991802215576172
+                },
+                {
+                    "layer": "layer.1.self_attn.v_proj",
+                    "ppl": 9.99177360534668
+                },
+                {
+                    "layer": "layer.1.self_attn.o_proj",
+                    "ppl": 9.989343643188477
+                },
+                {
+                    "layer": "layer.1.mlp.gate_proj",
+                    "ppl": 9.991896629333496
+                },
+                {
+                    "layer": "layer.1.mlp.up_proj",
+                    "ppl": 9.992478370666504
+                },
+                {
+                    "layer": "layer.1.mlp.down_proj",
+                    "ppl": 9.9921875
+                },
+                {
+                    "layer": "layer.2.self_attn.q_proj",
+                    "ppl": 9.991708755493164
+                },
+                {
+                    "layer": "layer.2.self_attn.k_proj",
+                    "ppl": 9.99179744720459
+                },
+                {
+                    "layer": "layer.2.self_attn.v_proj",
+                    "ppl": 9.991837501525879
+                },
+                {
+                    "layer": "layer.2.self_attn.o_proj",
+                    "ppl": 9.991842269897461
+                },
+                {
+                    "layer": "layer.2.mlp.gate_proj",
+                    "ppl": 9.992039680480957
+                },
+                {
+                    "layer": "layer.2.mlp.up_proj",
+                    "ppl": 9.991783142089844
+                },
+                {
+                    "layer": "layer.2.mlp.down_proj",
+                    "ppl": 9.991183280944824
+                },
+                {
+                    "layer": "layer.3.self_attn.q_proj",
+                    "ppl": 9.991918563842773
+                },
+                {
+                    "layer": "layer.3.self_attn.k_proj",
+                    "ppl": 9.991430282592773
+                },
+                {
+                    "layer": "layer.3.self_attn.v_proj",
+                    "ppl": 9.991148948669434
+                },
+                {
+                    "layer": "layer.3.self_attn.o_proj",
+                    "ppl": 9.99170207977295
+                },
+                {
+                    "layer": "layer.3.mlp.gate_proj",
+                    "ppl": 9.991687774658203
+                },
+                {
+                    "layer": "layer.3.mlp.up_proj",
+                    "ppl": 9.991859436035156
+                },
+                {
+                    "layer": "layer.3.mlp.down_proj",
+                    "ppl": 9.991928100585938
+                },
+                {
+                    "layer": "layer.4.self_attn.q_proj",
+                    "ppl": 9.99184513092041
+                },
+                {
+                    "layer": "layer.4.self_attn.k_proj",
+                    "ppl": 9.992378234863281
+                },
+                {
+                    "layer": "layer.4.self_attn.v_proj",
+                    "ppl": 9.991939544677734
+                },
+                {
+                    "layer": "layer.4.self_attn.o_proj",
+                    "ppl": 9.991907119750977
+                },
+                {
+                    "layer": "layer.4.mlp.gate_proj",
+                    "ppl": 9.991008758544922
+                },
+                {
+                    "layer": "layer.4.mlp.up_proj",
+                    "ppl": 9.99181079864502
+                },
+                {
+                    "layer": "layer.4.mlp.down_proj",
+                    "ppl": 9.992833137512207
+                },
+                {
+                    "layer": "layer.5.self_attn.q_proj",
+                    "ppl": 9.991840362548828
+                },
+                {
+                    "layer": "layer.5.self_attn.k_proj",
+                    "ppl": 9.991721153259277
+                },
+                {
+                    "layer": "layer.5.self_attn.v_proj",
+                    "ppl": 9.991996765136719
+                },
+                {
+                    "layer": "layer.5.self_attn.o_proj",
+                    "ppl": 9.991945266723633
+                },
+                {
+                    "layer": "layer.5.mlp.gate_proj",
+                    "ppl": 9.990937232971191
+                },
+                {
+                    "layer": "layer.5.mlp.up_proj",
+                    "ppl": 9.990982055664062
+                },
+                {
+                    "layer": "layer.5.mlp.down_proj",
+                    "ppl": 9.99046516418457
+                },
+                {
+                    "layer": "layer.6.self_attn.q_proj",
+                    "ppl": 9.992035865783691
+                },
+                {
+                    "layer": "layer.6.self_attn.k_proj",
+                    "ppl": 9.992044448852539
+                },
+                {
+                    "layer": "layer.6.self_attn.v_proj",
+                    "ppl": 9.991878509521484
+                },
+                {
+                    "layer": "layer.6.self_attn.o_proj",
+                    "ppl": 9.992192268371582
+                },
+                {
+                    "layer": "layer.6.mlp.gate_proj",
+                    "ppl": 9.991172790527344
+                },
+                {
+                    "layer": "layer.6.mlp.up_proj",
+                    "ppl": 9.992650032043457
+                },
+                {
+                    "layer": "layer.6.mlp.down_proj",
+                    "ppl": 9.99230670928955
+                },
+                {
+                    "layer": "layer.7.self_attn.q_proj",
+                    "ppl": 9.992268562316895
+                },
+                {
+                    "layer": "layer.7.self_attn.k_proj",
+                    "ppl": 9.9916353225708
+                },
+                {
+                    "layer": "layer.7.self_attn.v_proj",
+                    "ppl": 9.99170207977295
+                },
+                {
+                    "layer": "layer.7.self_attn.o_proj",
+                    "ppl": 9.991859436035156
+                },
+                {
+                    "layer": "layer.7.mlp.gate_proj",
+                    "ppl": 9.991716384887695
+                },
+                {
+                    "layer": "layer.7.mlp.up_proj",
+                    "ppl": 9.990920066833496
+                },
+                {
+                    "layer": "layer.7.mlp.down_proj",
+                    "ppl": 9.992217063903809
+                },
+                {
+                    "layer": "layer.8.self_attn.q_proj",
+                    "ppl": 9.992116928100586
+                },
+                {
+                    "layer": "layer.8.self_attn.k_proj",
+                    "ppl": 9.991764068603516
+                },
+                {
+                    "layer": "layer.8.self_attn.v_proj",
+                    "ppl": 9.99203109741211
+                },
+                {
+                    "layer": "layer.8.self_attn.o_proj",
+                    "ppl": 9.99207592010498
+                },
+                {
+                    "layer": "layer.8.mlp.gate_proj",
+                    "ppl": 9.991268157958984
+                },
+                {
+                    "layer": "layer.8.mlp.up_proj",
+                    "ppl": 9.99142074584961
+                },
+                {
+                    "layer": "layer.8.mlp.down_proj",
+                    "ppl": 9.991275787353516
+                },
+                {
+                    "layer": "layer.9.self_attn.q_proj",
+                    "ppl": 9.99223518371582
+                },
+                {
+                    "layer": "layer.9.self_attn.k_proj",
+                    "ppl": 9.991826057434082
+                },
+                {
+                    "layer": "layer.9.self_attn.v_proj",
+                    "ppl": 9.99184513092041
+                },
+                {
+                    "layer": "layer.9.self_attn.o_proj",
+                    "ppl": 9.991172790527344
+                },
+                {
+                    "layer": "layer.9.mlp.gate_proj",
+                    "ppl": 9.991533279418945
+                },
+                {
+                    "layer": "layer.9.mlp.up_proj",
+                    "ppl": 9.991816520690918
+                },
+                {
+                    "layer": "layer.9.mlp.down_proj",
+                    "ppl": 9.9908447265625
+                },
+                {
+                    "layer": "layer.10.self_attn.q_proj",
+                    "ppl": 9.991678237915039
+                },
+                {
+                    "layer": "layer.10.self_attn.k_proj",
+                    "ppl": 9.992087364196777
+                },
+                {
+                    "layer": "layer.10.self_attn.v_proj",
+                    "ppl": 9.991971015930176
+                },
+                {
+                    "layer": "layer.10.self_attn.o_proj",
+                    "ppl": 9.99221420288086
+                },
+                {
+                    "layer": "layer.10.mlp.gate_proj",
+                    "ppl": 9.992042541503906
+                },
+                {
+                    "layer": "layer.10.mlp.up_proj",
+                    "ppl": 9.992419242858887
+                },
+                {
+                    "layer": "layer.10.mlp.down_proj",
+                    "ppl": 9.992709159851074
+                },
+                {
+                    "layer": "layer.11.self_attn.q_proj",
+                    "ppl": 9.992268562316895
+                },
+                {
+                    "layer": "layer.11.self_attn.k_proj",
+                    "ppl": 9.992128372192383
+                },
+                {
+                    "layer": "layer.11.self_attn.v_proj",
+                    "ppl": 9.992077827453613
+                },
+                {
+                    "layer": "layer.11.self_attn.o_proj",
+                    "ppl": 9.992120742797852
+                },
+                {
+                    "layer": "layer.11.mlp.gate_proj",
+                    "ppl": 9.991851806640625
+                },
+                {
+                    "layer": "layer.11.mlp.up_proj",
+                    "ppl": 9.991659164428711
+                },
+                {
+                    "layer": "layer.11.mlp.down_proj",
+                    "ppl": 9.990784645080566
+                },
+                {
+                    "layer": "layer.12.self_attn.q_proj",
+                    "ppl": 9.99179458618164
+                },
+                {
+                    "layer": "layer.12.self_attn.k_proj",
+                    "ppl": 9.99186897277832
+                },
+                {
+                    "layer": "layer.12.self_attn.v_proj",
+                    "ppl": 9.992042541503906
+                },
+                {
+                    "layer": "layer.12.self_attn.o_proj",
+                    "ppl": 9.99133014678955
+                },
+                {
+                    "layer": "layer.12.mlp.gate_proj",
+                    "ppl": 9.991622924804688
+                },
+                {
+                    "layer": "layer.12.mlp.up_proj",
+                    "ppl": 9.99203109741211
+                },
+                {
+                    "layer": "layer.12.mlp.down_proj",
+                    "ppl": 9.991732597351074
+                },
+                {
+                    "layer": "layer.13.self_attn.q_proj",
+                    "ppl": 9.992106437683105
+                },
+                {
+                    "layer": "layer.13.self_attn.k_proj",
+                    "ppl": 9.992217063903809
+                },
+                {
+                    "layer": "layer.13.self_attn.v_proj",
+                    "ppl": 9.991859436035156
+                },
+                {
+                    "layer": "layer.13.self_attn.o_proj",
+                    "ppl": 9.99156379699707
+                },
+                {
+                    "layer": "layer.13.mlp.gate_proj",
+                    "ppl": 9.99118709564209
+                },
+                {
+                    "layer": "layer.13.mlp.up_proj",
+                    "ppl": 9.991016387939453
+                },
+                {
+                    "layer": "layer.13.mlp.down_proj",
+                    "ppl": 9.991382598876953
+                },
+                {
+                    "layer": "layer.14.self_attn.q_proj",
+                    "ppl": 9.992112159729004
+                },
+                {
+                    "layer": "layer.14.self_attn.k_proj",
+                    "ppl": 9.991692543029785
+                },
+                {
+                    "layer": "layer.14.self_attn.v_proj",
+                    "ppl": 9.991945266723633
+                },
+                {
+                    "layer": "layer.14.self_attn.o_proj",
+                    "ppl": 9.99097728729248
+                },
+                {
+                    "layer": "layer.14.mlp.gate_proj",
+                    "ppl": 9.992411613464355
+                },
+                {
+                    "layer": "layer.14.mlp.up_proj",
+                    "ppl": 9.991988182067871
+                },
+                {
+                    "layer": "layer.14.mlp.down_proj",
+                    "ppl": 9.992793083190918
+                },
+                {
+                    "layer": "layer.15.self_attn.q_proj",
+                    "ppl": 9.991853713989258
+                },
+                {
+                    "layer": "layer.15.self_attn.k_proj",
+                    "ppl": 9.991958618164062
+                },
+                {
+                    "layer": "layer.15.self_attn.v_proj",
+                    "ppl": 9.991740226745605
+                },
+                {
+                    "layer": "layer.15.self_attn.o_proj",
+                    "ppl": 9.99128246307373
+                },
+                {
+                    "layer": "layer.15.mlp.gate_proj",
+                    "ppl": 9.992297172546387
+                },
+                {
+                    "layer": "layer.15.mlp.up_proj",
+                    "ppl": 9.991775512695312
+                },
+                {
+                    "layer": "layer.15.mlp.down_proj",
+                    "ppl": 9.992101669311523
+                },
+                {
+                    "layer": "layer.16.self_attn.q_proj",
+                    "ppl": 9.991930961608887
+                },
+                {
+                    "layer": "layer.16.self_attn.k_proj",
+                    "ppl": 9.991888046264648
+                },
+                {
+                    "layer": "layer.16.self_attn.v_proj",
+                    "ppl": 9.991928100585938
+                },
+                {
+                    "layer": "layer.16.self_attn.o_proj",
+                    "ppl": 9.992283821105957
+                },
+                {
+                    "layer": "layer.16.mlp.gate_proj",
+                    "ppl": 9.9921875
+                },
+                {
+                    "layer": "layer.16.mlp.up_proj",
+                    "ppl": 9.9918851852417
+                },
+                {
+                    "layer": "layer.16.mlp.down_proj",
+                    "ppl": 9.992573738098145
+                },
+                {
+                    "layer": "layer.17.self_attn.q_proj",
+                    "ppl": 9.991878509521484
+                },
+                {
+                    "layer": "layer.17.self_attn.k_proj",
+                    "ppl": 9.991842269897461
+                },
+                {
+                    "layer": "layer.17.self_attn.v_proj",
+                    "ppl": 9.99163818359375
+                },
+                {
+                    "layer": "layer.17.self_attn.o_proj",
+                    "ppl": 9.992416381835938
+                },
+                {
+                    "layer": "layer.17.mlp.gate_proj",
+                    "ppl": 9.99181842803955
+                },
+                {
+                    "layer": "layer.17.mlp.up_proj",
+                    "ppl": 9.992230415344238
+                },
+                {
+                    "layer": "layer.17.mlp.down_proj",
+                    "ppl": 9.992159843444824
+                },
+                {
+                    "layer": "layer.18.self_attn.q_proj",
+                    "ppl": 9.991654396057129
+                },
+                {
+                    "layer": "layer.18.self_attn.k_proj",
+                    "ppl": 9.992087364196777
+                },
+                {
+                    "layer": "layer.18.self_attn.v_proj",
+                    "ppl": 9.992199897766113
+                },
+                {
+                    "layer": "layer.18.self_attn.o_proj",
+                    "ppl": 9.9927339553833
+                },
+                {
+                    "layer": "layer.18.mlp.gate_proj",
+                    "ppl": 9.991934776306152
+                },
+                {
+                    "layer": "layer.18.mlp.up_proj",
+                    "ppl": 9.992147445678711
+                },
+                {
+                    "layer": "layer.18.mlp.down_proj",
+                    "ppl": 9.992630958557129
+                },
+                {
+                    "layer": "layer.19.self_attn.q_proj",
+                    "ppl": 9.992015838623047
+                },
+                {
+                    "layer": "layer.19.self_attn.k_proj",
+                    "ppl": 9.992009162902832
+                },
+                {
+                    "layer": "layer.19.self_attn.v_proj",
+                    "ppl": 9.992206573486328
+                },
+                {
+                    "layer": "layer.19.self_attn.o_proj",
+                    "ppl": 9.992144584655762
+                },
+                {
+                    "layer": "layer.19.mlp.gate_proj",
+                    "ppl": 9.992080688476562
+                },
+                {
+                    "layer": "layer.19.mlp.up_proj",
+                    "ppl": 9.990987777709961
+                },
+                {
+                    "layer": "layer.19.mlp.down_proj",
+                    "ppl": 9.992058753967285
+                },
+                {
+                    "layer": "layer.20.self_attn.q_proj",
+                    "ppl": 9.991915702819824
+                },
+                {
+                    "layer": "layer.20.self_attn.k_proj",
+                    "ppl": 9.99203109741211
+                },
+                {
+                    "layer": "layer.20.self_attn.v_proj",
+                    "ppl": 9.992240905761719
+                },
+                {
+                    "layer": "layer.20.self_attn.o_proj",
+                    "ppl": 9.991654396057129
+                },
+                {
+                    "layer": "layer.20.mlp.gate_proj",
+                    "ppl": 9.991783142089844
+                },
+                {
+                    "layer": "layer.20.mlp.up_proj",
+                    "ppl": 9.99222183227539
+                },
+                {
+                    "layer": "layer.20.mlp.down_proj",
+                    "ppl": 9.991921424865723
+                },
+                {
+                    "layer": "layer.21.self_attn.q_proj",
+                    "ppl": 9.992063522338867
+                },
+                {
+                    "layer": "layer.21.self_attn.k_proj",
+                    "ppl": 9.991921424865723
+                },
+                {
+                    "layer": "layer.21.self_attn.v_proj",
+                    "ppl": 9.991896629333496
+                },
+                {
+                    "layer": "layer.21.self_attn.o_proj",
+                    "ppl": 9.991975784301758
+                },
+                {
+                    "layer": "layer.21.mlp.gate_proj",
+                    "ppl": 9.992283821105957
+                },
+                {
+                    "layer": "layer.21.mlp.up_proj",
+                    "ppl": 9.991435050964355
+                },
+                {
+                    "layer": "layer.21.mlp.down_proj",
+                    "ppl": 9.991559028625488
+                },
+                {
+                    "layer": "layer.22.self_attn.q_proj",
+                    "ppl": 9.992159843444824
+                },
+                {
+                    "layer": "layer.22.self_attn.k_proj",
+                    "ppl": 9.992247581481934
+                },
+                {
+                    "layer": "layer.22.self_attn.v_proj",
+                    "ppl": 9.9920072555542
+                },
+                {
+                    "layer": "layer.22.self_attn.o_proj",
+                    "ppl": 9.99223518371582
+                },
+                {
+                    "layer": "layer.22.mlp.gate_proj",
+                    "ppl": 9.991992950439453
+                },
+                {
+                    "layer": "layer.22.mlp.up_proj",
+                    "ppl": 9.992273330688477
+                },
+                {
+                    "layer": "layer.22.mlp.down_proj",
+                    "ppl": 9.991662979125977
+                },
+                {
+                    "layer": "layer.23.self_attn.q_proj",
+                    "ppl": 9.992020606994629
+                },
+                {
+                    "layer": "layer.23.self_attn.k_proj",
+                    "ppl": 9.991930961608887
+                },
+                {
+                    "layer": "layer.23.self_attn.v_proj",
+                    "ppl": 9.992112159729004
+                },
+                {
+                    "layer": "layer.23.self_attn.o_proj",
+                    "ppl": 9.991926193237305
+                },
+                {
+                    "layer": "layer.23.mlp.gate_proj",
+                    "ppl": 9.992077827453613
+                },
+                {
+                    "layer": "layer.23.mlp.up_proj",
+                    "ppl": 9.9923095703125
+                },
+                {
+                    "layer": "layer.23.mlp.down_proj",
+                    "ppl": 9.991737365722656
+                },
+                {
+                    "layer": "layer.24.self_attn.q_proj",
+                    "ppl": 9.99181842803955
+                },
+                {
+                    "layer": "layer.24.self_attn.k_proj",
+                    "ppl": 9.99209976196289
+                },
+                {
+                    "layer": "layer.24.self_attn.v_proj",
+                    "ppl": 9.991910934448242
+                },
+                {
+                    "layer": "layer.24.self_attn.o_proj",
+                    "ppl": 9.99153995513916
+                },
+                {
+                    "layer": "layer.24.mlp.gate_proj",
+                    "ppl": 9.991697311401367
+                },
+                {
+                    "layer": "layer.24.mlp.up_proj",
+                    "ppl": 9.992085456848145
+                },
+                {
+                    "layer": "layer.24.mlp.down_proj",
+                    "ppl": 9.991140365600586
+                },
+                {
+                    "layer": "layer.25.self_attn.q_proj",
+                    "ppl": 9.992074012756348
+                },
+                {
+                    "layer": "layer.25.self_attn.k_proj",
+                    "ppl": 9.991950035095215
+                },
+                {
+                    "layer": "layer.25.self_attn.v_proj",
+                    "ppl": 9.991775512695312
+                },
+                {
+                    "layer": "layer.25.self_attn.o_proj",
+                    "ppl": 9.992178916931152
+                },
+                {
+                    "layer": "layer.25.mlp.gate_proj",
+                    "ppl": 9.99125862121582
+                },
+                {
+                    "layer": "layer.25.mlp.up_proj",
+                    "ppl": 9.992454528808594
+                },
+                {
+                    "layer": "layer.25.mlp.down_proj",
+                    "ppl": 9.99123764038086
+                },
+                {
+                    "layer": "layer.26.self_attn.q_proj",
+                    "ppl": 9.991673469543457
+                },
+                {
+                    "layer": "layer.26.self_attn.k_proj",
+                    "ppl": 9.99202823638916
+                },
+                {
+                    "layer": "layer.26.self_attn.v_proj",
+                    "ppl": 9.991668701171875
+                },
+                {
+                    "layer": "layer.26.self_attn.o_proj",
+                    "ppl": 9.99179744720459
+                },
+                {
+                    "layer": "layer.26.mlp.gate_proj",
+                    "ppl": 9.990954399108887
+                },
+                {
+                    "layer": "layer.26.mlp.up_proj",
+                    "ppl": 9.991511344909668
+                },
+                {
+                    "layer": "layer.26.mlp.down_proj",
+                    "ppl": 9.991479873657227
+                },
+                {
+                    "layer": "layer.27.self_attn.q_proj",
+                    "ppl": 9.992050170898438
+                },
+                {
+                    "layer": "layer.27.self_attn.k_proj",
+                    "ppl": 9.9920015335083
+                },
+                {
+                    "layer": "layer.27.self_attn.v_proj",
+                    "ppl": 9.992063522338867
+                },
+                {
+                    "layer": "layer.27.self_attn.o_proj",
+                    "ppl": 9.992464065551758
+                },
+                {
+                    "layer": "layer.27.mlp.gate_proj",
+                    "ppl": 9.990560531616211
+                },
+                {
+                    "layer": "layer.27.mlp.up_proj",
+                    "ppl": 9.989849090576172
+                },
+                {
+                    "layer": "layer.27.mlp.down_proj",
+                    "ppl": 9.991116523742676
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file