| { |
| "original_ppl": 9.97196102142334, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 83332.109375 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 81348.953125 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 25781.796875 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 27834.796875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 62059.78125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 64951.6875 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 62222.87109375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 85301.765625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 86110.703125 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 45133.1328125 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 36500.11328125 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 77381.359375 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 49991.375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 46378.9609375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 88139.484375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 90501.71875 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 120570.203125 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 50978.390625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 78643.28125 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 82742.3125 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 64036.484375 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 76686.3828125 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 81167.9296875 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 87316.3125 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 60217.55078125 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 59247.91015625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 62741.40234375 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 84264.7265625 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 72960.6640625 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 93884.59375 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 77069.7578125 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 92530.4609375 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 101563.9921875 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 50916.39453125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 89985.078125 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 86380.2421875 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 68246.171875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 97836.4609375 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 67856.2578125 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 92100.484375 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 100485.90625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 41168.40234375 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 79279.359375 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 82641.6796875 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 63274.9296875 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 52274.33203125 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 60532.96875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 100568.0625 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 78218.3515625 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 88200.6171875 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 88442.1875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 104099.015625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 139569.671875 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 53775.09375 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 56642.953125 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 70138.421875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 76445.34375 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 82595.5078125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 108704.234375 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 76762.40625 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 90175.453125 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 60534.23828125 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 52055.08984375 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 83108.46875 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 83471.1484375 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 136432.921875 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 152531.609375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 74755.015625 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 76557.9921875 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 133054.65625 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 97931.8671875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 91111.8046875 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 70577.2421875 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 34539.39453125 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 78766.8984375 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 93544.703125 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 71034.9375 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 91325.03125 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 85825.4765625 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 79586.921875 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 82422.2421875 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 51500.8125 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 67704.21875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 64296.7421875 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 85187.3828125 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 85370.0546875 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 45587.73828125 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 45965.9296875 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 97300.6953125 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 77209.828125 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 58815.9453125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 85948.0078125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 79145.953125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 51608.9296875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 85309.3359375 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 82983.8125 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 69577.9375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 76646.3125 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 86569.8359375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 86495.3203125 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 78097.0078125 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 89474.8125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 62052.5 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 121601.6328125 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 65683.3515625 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 90000.015625 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 85875.5 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 51253.1796875 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 60614.70703125 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 69880.375 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 82342.890625 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 77203.5703125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 86468.59375 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 85792.90625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 55330.50390625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 78712.8359375 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 86215.640625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 102103.765625 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 82107.328125 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 81355.1640625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 85610.15625 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 73543.0 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 56332.01171875 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 84936.15625 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 80587.7578125 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 76033.25 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 85493.8125 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 89435.65625 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 94210.34375 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 56984.73046875 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 79825.453125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 59850.96484375 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 87420.8828125 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 87652.3828125 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 83483.0859375 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 103584.7421875 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 56013.58984375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 100148.1875 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 91430.125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 84644.1640625 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 86207.0078125 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 85745.3046875 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 103126.3125 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 79577.7421875 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 87526.578125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 89423.28125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 82647.515625 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 85887.8671875 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 82824.2578125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 104417.4765625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 69490.734375 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 101418.03125 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 85467.7265625 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 73187.4296875 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 85862.8828125 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 86954.09375 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 88807.8125 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 70540.1015625 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 67607.828125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 74690.4453125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 99885.3125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 92043.3125 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 82548.1015625 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 74932.453125 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 68666.203125 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 83727.390625 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 83494.7109375 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 100388.8671875 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 86749.25 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 83572.3125 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 82877.3515625 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 89907.109375 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 85133.375 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 94788.8359375 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 84814.09375 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 83655.2421875 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 85422.8203125 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 103965.4765625 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 86323.90625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 101230.765625 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 83441.78125 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 92780.625 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 85717.34375 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 85034.953125 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 85586.15625 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 75896.609375 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 87989.3984375 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 78273.125 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 88496.859375 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 86038.546875 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 84773.5 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 89991.34375 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 78720.7890625 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 108667.953125 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 89630.0703125 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 78953.484375 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 168.69410705566406 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 167.3025665283203 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 134.85409545898438 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 147.0169219970703 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 153.91915893554688 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 157.78953552246094 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 153.5771484375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 168.2405548095703 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 168.6426239013672 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 163.9848175048828 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 175.6777801513672 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 158.56808471679688 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 169.42819213867188 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 162.0041046142578 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 168.2506561279297 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 168.59390258789062 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 158.20021057128906 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 152.7701416015625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 144.7792510986328 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 168.1245880126953 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 166.7340087890625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 167.2913055419922 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 169.46438598632812 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 159.04104614257812 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 158.66580200195312 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 150.30787658691406 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 164.19912719726562 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 160.14694213867188 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 163.7737579345703 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 168.6818084716797 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 169.0232696533203 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 163.02841186523438 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 152.95265197753906 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 165.4967498779297 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 173.02493286132812 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 169.71926879882812 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 168.32321166992188 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 171.78506469726562 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 172.12879943847656 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 156.8002471923828 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 169.80587768554688 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 168.16331481933594 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 171.93313598632812 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 170.42822265625 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 166.3150634765625 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 164.08706665039062 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 159.70425415039062 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 165.3900909423828 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 162.13487243652344 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 170.3074188232422 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 168.4875030517578 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 163.2733917236328 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 156.7130889892578 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 150.77174377441406 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 159.7360076904297 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 165.3878936767578 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 173.43414306640625 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 169.6393280029297 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 167.9151611328125 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 167.08619689941406 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 160.76515197753906 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 162.380126953125 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 165.14808654785156 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 168.6042022705078 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 167.47824096679688 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 168.47898864746094 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 163.73831176757812 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 166.50613403320312 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 165.7300262451172 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 168.846923828125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 168.58209228515625 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 167.91220092773438 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 174.09750366210938 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 167.61534118652344 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 169.89886474609375 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 169.2921142578125 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 169.01593017578125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 167.5283966064453 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 168.47947692871094 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 165.90406799316406 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 165.99972534179688 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 165.912841796875 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 172.89553833007812 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 170.34291076660156 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 170.30311584472656 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 168.70327758789062 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 169.1955108642578 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 165.52769470214844 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 167.78005981445312 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 172.9617462158203 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 170.7941436767578 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 169.14678955078125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 167.37380981445312 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 169.5671844482422 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 164.41168212890625 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 162.4380645751953 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 170.59213256835938 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 176.98516845703125 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 169.9107666015625 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 168.72178649902344 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 168.07215881347656 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 170.56326293945312 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 169.74880981445312 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 170.60238647460938 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 168.23294067382812 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 168.81463623046875 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 168.92770385742188 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 169.9482879638672 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 166.65753173828125 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 167.7421417236328 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 170.9343719482422 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 166.71420288085938 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 168.89065551757812 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 169.56088256835938 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 170.41140747070312 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 167.99749755859375 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 166.1726837158203 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 169.96035766601562 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 167.08116149902344 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 169.047119140625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 167.61502075195312 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 167.5784149169922 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 175.95620727539062 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 171.89068603515625 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 167.10482788085938 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 166.0119171142578 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 169.005859375 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 170.59197998046875 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 168.8288116455078 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 166.205810546875 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 166.53225708007812 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 167.81149291992188 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 171.29623413085938 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 170.4592742919922 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 169.1224365234375 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 174.17156982421875 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 171.35145568847656 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 168.1353302001953 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 167.7538299560547 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 168.72451782226562 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 169.44894409179688 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 169.5083465576172 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 168.2606201171875 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 169.7484130859375 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 169.15194702148438 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 168.19691467285156 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 167.42849731445312 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 169.51043701171875 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 169.20535278320312 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 168.32289123535156 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 173.6912841796875 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 168.3231201171875 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 167.8394317626953 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 169.46389770507812 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 169.63949584960938 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 169.69911193847656 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 168.16082763671875 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 165.98532104492188 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 167.897705078125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 168.83944702148438 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 166.7910919189453 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 169.135498046875 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 167.2165069580078 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 168.8903350830078 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 167.9126739501953 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 167.8650360107422 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 168.4675750732422 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 166.85760498046875 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 169.2283477783203 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 168.97378540039062 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 167.07391357421875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 167.07662963867188 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 167.44293212890625 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 169.12307739257812 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 167.47384643554688 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 169.68487548828125 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 168.2967987060547 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 171.37777709960938 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 170.510498046875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 168.1109619140625 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 168.1544189453125 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 168.1858367919922 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 169.4444122314453 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 167.9197998046875 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 168.7620086669922 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 168.62879943847656 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 168.68331909179688 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 167.77685546875 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 169.30543518066406 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 169.27587890625 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 168.8433074951172 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 169.55361938476562 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 167.65817260742188 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 165.5074920654297 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 167.4623565673828 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 165.63079833984375 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 16.816091537475586 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 16.80735969543457 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 16.645936965942383 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 16.93777847290039 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 16.7054443359375 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 16.74637794494629 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 16.680191040039062 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 16.808029174804688 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 16.817955017089844 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 16.794597625732422 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 17.078054428100586 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 16.684486389160156 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 16.77771759033203 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 16.807056427001953 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 16.829391479492188 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 16.85578155517578 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 16.81163215637207 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 16.761272430419922 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 16.623249053955078 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 16.745830535888672 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 16.735363006591797 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 16.834251403808594 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 16.811296463012695 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 16.80232810974121 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 16.77528953552246 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 16.66208267211914 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 16.772022247314453 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 16.740541458129883 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 16.826749801635742 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 16.81400489807129 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 16.795495986938477 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 16.771345138549805 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 16.66862678527832 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 16.704448699951172 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 16.728321075439453 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 16.83759307861328 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 16.834495544433594 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 16.800758361816406 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 16.80919647216797 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 16.59544563293457 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 16.76308250427246 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 16.8048038482666 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 16.914257049560547 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 16.84715461730957 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 16.825233459472656 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 16.83127212524414 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 16.668018341064453 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 16.790830612182617 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 16.75554084777832 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 16.834848403930664 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 16.82583236694336 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 16.790273666381836 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 16.79183578491211 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 16.564863204956055 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 16.64543342590332 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 16.75745391845703 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 16.867660522460938 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 16.824426651000977 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 16.823348999023438 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 16.83637809753418 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 16.6285400390625 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 16.746749877929688 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 16.785511016845703 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 16.826574325561523 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 16.831188201904297 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 16.818683624267578 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 16.809307098388672 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 16.768367767333984 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 16.749040603637695 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 16.7552433013916 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 16.843311309814453 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 16.82830047607422 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 16.85602569580078 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 16.78997039794922 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 16.79315757751465 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 16.79081153869629 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 16.817453384399414 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 16.802064895629883 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 16.813669204711914 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 16.777050018310547 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 16.724058151245117 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 16.715961456298828 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 16.78472328186035 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 16.784912109375 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 16.84333038330078 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 16.839614868164062 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 16.82872772216797 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 16.828916549682617 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 16.753414154052734 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 16.804367065429688 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 16.81096649169922 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 16.81706428527832 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 16.80708885192871 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 16.78807258605957 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 16.7968807220459 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 16.651710510253906 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 16.751850128173828 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 16.83733367919922 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 16.855628967285156 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 16.83863639831543 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 16.828649520874023 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 16.83321762084961 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 16.75590705871582 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 16.78203010559082 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 16.800777435302734 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 16.81707000732422 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 16.81329345703125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 16.830177307128906 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 16.814172744750977 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 16.749544143676758 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 16.782508850097656 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 16.79301643371582 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 16.821210861206055 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 16.835525512695312 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 16.794036865234375 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 16.860326766967773 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 16.795434951782227 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 16.793045043945312 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 16.842557907104492 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 16.829526901245117 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 16.826160430908203 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 16.854816436767578 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 16.840280532836914 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 16.797454833984375 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 16.827211380004883 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 16.821142196655273 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 16.816619873046875 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 16.832456588745117 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 16.801607131958008 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 16.806598663330078 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 16.775604248046875 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 16.771047592163086 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 16.798187255859375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 16.866594314575195 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 16.800806045532227 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 16.85487174987793 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 16.834245681762695 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 16.771350860595703 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 16.776689529418945 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 16.772886276245117 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 16.834583282470703 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 16.827171325683594 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 16.811311721801758 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 16.786184310913086 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 16.795663833618164 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 16.79833221435547 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 16.78153419494629 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 16.836599349975586 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 16.819507598876953 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 16.8144588470459 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 16.851163864135742 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 16.78438377380371 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 16.80192756652832 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 16.793245315551758 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 16.824665069580078 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 16.825010299682617 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 16.813615798950195 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 16.798166275024414 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 16.749866485595703 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 16.7833309173584 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 16.74230194091797 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 16.831605911254883 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 16.823076248168945 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 16.813953399658203 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 16.849159240722656 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 16.76750373840332 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 16.77958869934082 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 16.73868179321289 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 16.82379722595215 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 16.818166732788086 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 16.821287155151367 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 16.840299606323242 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 16.7641544342041 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 16.766313552856445 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 16.754161834716797 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 16.83121109008789 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 16.81581687927246 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 16.803930282592773 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 16.8447265625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 16.75250816345215 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 16.766193389892578 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 16.7667293548584 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 16.832719802856445 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 16.822650909423828 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 16.823837280273438 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 16.85397720336914 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 16.7725772857666 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 16.75528335571289 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 16.76927947998047 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 16.828399658203125 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 16.81958770751953 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 16.833404541015625 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 16.843053817749023 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 16.69053840637207 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 16.710926055908203 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 16.69918441772461 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 12.195830345153809 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 12.198372840881348 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 12.180115699768066 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 12.190705299377441 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 12.178506851196289 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 12.185070991516113 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 12.172568321228027 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 12.194560050964355 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 12.19727611541748 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 12.191821098327637 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 12.202656745910645 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 12.166489601135254 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 12.192140579223633 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 12.187891960144043 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 12.200326919555664 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 12.192219734191895 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 12.193717002868652 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 12.198744773864746 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 12.174346923828125 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 12.183810234069824 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 12.186564445495605 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 12.199477195739746 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 12.191658020019531 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 12.189967155456543 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 12.199512481689453 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 12.182425498962402 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 12.192390441894531 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 12.184161186218262 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 12.196310043334961 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 12.20107364654541 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 12.187850952148438 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 12.196086883544922 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 12.170602798461914 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 12.188851356506348 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 12.183006286621094 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 12.201290130615234 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 12.201272010803223 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 12.194780349731445 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 12.184701919555664 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 12.158903121948242 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 12.186938285827637 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 12.186903953552246 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 12.204073905944824 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 12.193321228027344 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 12.194419860839844 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 12.200675964355469 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 12.172024726867676 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 12.184027671813965 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 12.187215805053711 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 12.200061798095703 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 12.199942588806152 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 12.194507598876953 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 12.194100379943848 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 12.16766357421875 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 12.170788764953613 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 12.181585311889648 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 12.196106910705566 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 12.201425552368164 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 12.198567390441895 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 12.1928129196167 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 12.175227165222168 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 12.192557334899902 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 12.189879417419434 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 12.200451850891113 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 12.19792652130127 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 12.19448471069336 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 12.190045356750488 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 12.183615684509277 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 12.18881607055664 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 12.189757347106934 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 12.198962211608887 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 12.197670936584473 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 12.194870948791504 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 12.201330184936523 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 12.187311172485352 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 12.189641952514648 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 12.189981460571289 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 12.197240829467773 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 12.197179794311523 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 12.18908405303955 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 12.17735481262207 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 12.18508243560791 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 12.19332504272461 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 12.19530200958252 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 12.206258773803711 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 12.201440811157227 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 12.199142456054688 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 12.19406795501709 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 12.192123413085938 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 12.196572303771973 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 12.186360359191895 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 12.196149826049805 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 12.196839332580566 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 12.19229507446289 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 12.200652122497559 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 12.170910835266113 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 12.187363624572754 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 12.190469741821289 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 12.197463989257812 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 12.201760292053223 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 12.199959754943848 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 12.195999145507812 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 12.195347785949707 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 12.187578201293945 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 12.196464538574219 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 12.196438789367676 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 12.197836875915527 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 12.197795867919922 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 12.203509330749512 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 12.191859245300293 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 12.186918258666992 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 12.19727611541748 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 12.19888687133789 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 12.199085235595703 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 12.201562881469727 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 12.187944412231445 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 12.192870140075684 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 12.1880464553833 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 12.190269470214844 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 12.19958209991455 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 12.198648452758789 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 12.194239616394043 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 12.202900886535645 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 12.194214820861816 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 12.19281005859375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 12.194185256958008 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 12.198107719421387 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 12.194565773010254 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 12.197409629821777 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 12.191821098327637 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 12.188043594360352 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 12.19308853149414 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 12.191277503967285 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 12.202199935913086 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 12.19740104675293 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 12.199410438537598 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 12.193265914916992 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 12.182888984680176 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 12.184446334838867 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 12.191447257995605 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 12.198471069335938 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 12.198386192321777 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 12.196746826171875 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 12.194589614868164 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 12.189786911010742 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 12.195269584655762 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 12.188821792602539 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 12.1997652053833 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 12.20077133178711 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 12.19654369354248 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 12.205243110656738 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 12.189764022827148 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 12.187328338623047 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 12.18155288696289 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 12.195489883422852 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 12.196496963500977 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 12.197467803955078 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 12.194589614868164 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 12.187393188476562 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 12.185384750366211 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 12.179558753967285 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 12.196786880493164 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 12.196386337280273 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 12.182453155517578 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 12.204495429992676 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 12.181593894958496 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 12.182836532592773 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 12.180176734924316 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 12.195624351501465 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 12.198346138000488 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 12.191450119018555 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 12.200736045837402 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 12.184626579284668 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 12.184115409851074 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 12.17769718170166 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 12.197446823120117 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 12.198479652404785 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 12.186229705810547 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 12.203984260559082 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 12.183870315551758 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 12.184196472167969 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 12.179142951965332 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 12.197104454040527 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 12.201415061950684 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 12.188841819763184 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 12.210712432861328 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 12.178765296936035 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 12.165717124938965 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 12.177505493164062 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 12.19913101196289 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 12.197893142700195 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 12.197391510009766 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 12.202534675598145 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 12.150505065917969 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 12.1620512008667 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 12.1500244140625 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 11.051345825195312 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 11.050370216369629 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 11.043753623962402 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 11.050064086914062 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 11.048126220703125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 11.046426773071289 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 11.049822807312012 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 11.050151824951172 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 11.051176071166992 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 11.04896068572998 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 11.059547424316406 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 11.046401023864746 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 11.048742294311523 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 11.049527168273926 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 11.051529884338379 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 11.05235767364502 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 11.048730850219727 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 11.04784107208252 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 11.042132377624512 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 11.052030563354492 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 11.045845031738281 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 11.051271438598633 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 11.049168586730957 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 11.053474426269531 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 11.048890113830566 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 11.044604301452637 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 11.052794456481934 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 11.047003746032715 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 11.050586700439453 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 11.049195289611816 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 11.048341751098633 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 11.051237106323242 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 11.042754173278809 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 11.0465669631958 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 11.051440238952637 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 11.051187515258789 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 11.051304817199707 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 11.051872253417969 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 11.050679206848145 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 11.039144515991211 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 11.04649829864502 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 11.045921325683594 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 11.053268432617188 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 11.052783966064453 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 11.052520751953125 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 11.050304412841797 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 11.043448448181152 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 11.048430442810059 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 11.047035217285156 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 11.052399635314941 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 11.05301284790039 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 11.048457145690918 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 11.04895305633545 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 11.040563583374023 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 11.050270080566406 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 11.047672271728516 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 11.053321838378906 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 11.051814079284668 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 11.050860404968262 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 11.051213264465332 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 11.04238224029541 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 11.046832084655762 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 11.04360580444336 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 11.051924705505371 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 11.052628517150879 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 11.051118850708008 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 11.050986289978027 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 11.047085762023926 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 11.05066204071045 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 11.049664497375488 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 11.051292419433594 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 11.050361633300781 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 11.05146598815918 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 11.051532745361328 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 11.04721450805664 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 11.049718856811523 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 11.051092147827148 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 11.051392555236816 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 11.049917221069336 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 11.051340103149414 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 11.048727035522461 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 11.045415878295898 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 11.04828929901123 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 11.049903869628906 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 11.054167747497559 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 11.052610397338867 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 11.052349090576172 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 11.051797866821289 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 11.05113410949707 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 11.050277709960938 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 11.053679466247559 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 11.050539016723633 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 11.051403045654297 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 11.051713943481445 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 11.046405792236328 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 11.043521881103516 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 11.0455322265625 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 11.050593376159668 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 11.054317474365234 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 11.050881385803223 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 11.052027702331543 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 11.053115844726562 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 11.050745010375977 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 11.047174453735352 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 11.050223350524902 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 11.050979614257812 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 11.048707962036133 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 11.054369926452637 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 11.045210838317871 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 11.04788875579834 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 11.050293922424316 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 11.051050186157227 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 11.051007270812988 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 11.050745010375977 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 11.050544738769531 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 11.052120208740234 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 11.05111312866211 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 11.052962303161621 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 11.05038070678711 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 11.052351951599121 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 11.0537748336792 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 11.0514554977417 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 11.052791595458984 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 11.050328254699707 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 11.050349235534668 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 11.049721717834473 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 11.051066398620605 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 11.051582336425781 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 11.05031681060791 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 11.051023483276367 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 11.049052238464355 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 11.043848991394043 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 11.048897743225098 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 11.053241729736328 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 11.051721572875977 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 11.053078651428223 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 11.053690910339355 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 11.047219276428223 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 11.046806335449219 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 11.045270919799805 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 11.05105209350586 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 11.052823066711426 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 11.051595687866211 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 11.047743797302246 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 11.049403190612793 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 11.050646781921387 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 11.047124862670898 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 11.051281929016113 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 11.050891876220703 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 11.050268173217773 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 11.053257942199707 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 11.046639442443848 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 11.050950050354004 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 11.050043106079102 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 11.050812721252441 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 11.048955917358398 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 11.050214767456055 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 11.049263954162598 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 11.043280601501465 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 11.048015594482422 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 11.049700736999512 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 11.050586700439453 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 11.049943923950195 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 11.049458503723145 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 11.052498817443848 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 11.046576499938965 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 11.048099517822266 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 11.042832374572754 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 11.051060676574707 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 11.051395416259766 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 11.05078125 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 11.052319526672363 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 11.04311752319336 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 11.045637130737305 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 11.043219566345215 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 11.05045223236084 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 11.0486421585083 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 11.052265167236328 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 11.049090385437012 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 11.043232917785645 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 11.044672966003418 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 11.043222427368164 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 11.05155086517334 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 11.050307273864746 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 11.049322128295898 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 11.054634094238281 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 11.042032241821289 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 11.047428131103516 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 11.046034812927246 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 11.051033973693848 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 11.050786972045898 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 11.049223899841309 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 11.057891845703125 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 11.026429176330566 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 11.032037734985352 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 11.031327247619629 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.676329612731934 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.673662185668945 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.67713451385498 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.675612449645996 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.675748825073242 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.674802780151367 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.674924850463867 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.676283836364746 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.676477432250977 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.675711631774902 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.67593765258789 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.672209739685059 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.676039695739746 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.673112869262695 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.676030158996582 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.676111221313477 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.676067352294922 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.67630386352539 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.672319412231445 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.672661781311035 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.673912048339844 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.676401138305664 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.67438793182373 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.676172256469727 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.675345420837402 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.671924591064453 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.675069808959961 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.67390251159668 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.676050186157227 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.677979469299316 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.676701545715332 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.6752347946167 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.67084789276123 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.672741889953613 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.674308776855469 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.676968574523926 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.676559448242188 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.675153732299805 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.675271034240723 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.672761917114258 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.67199420928955 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.676680564880371 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.6766939163208 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.676372528076172 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.6771240234375 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.6764497756958 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.672995567321777 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.676703453063965 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.674988746643066 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.675760269165039 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.677274703979492 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.677233695983887 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.675841331481934 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.671433448791504 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.67463493347168 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.6738862991333 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.677090644836426 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.676355361938477 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.67544937133789 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.678918838500977 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.670843124389648 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.673418045043945 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.675326347351074 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.67662525177002 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.677327156066895 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.674955368041992 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.677999496459961 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.676283836364746 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.674386024475098 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.67389965057373 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.677291870117188 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.676925659179688 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.677375793457031 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.67282485961914 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.675745010375977 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.67618465423584 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.676199913024902 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.67612361907959 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.675713539123535 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.675762176513672 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.674912452697754 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.675148963928223 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.674260139465332 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.67700481414795 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.676989555358887 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.677145004272461 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.675164222717285 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.677057266235352 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.674854278564453 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.676403999328613 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.673762321472168 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.676790237426758 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.676335334777832 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.676385879516602 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.675676345825195 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.671753883361816 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.672537803649902 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.67525577545166 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.67575740814209 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.67612361907959 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.676294326782227 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.677987098693848 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.676390647888184 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.675734519958496 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.676436424255371 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.676431655883789 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.676065444946289 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.676050186157227 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.677923202514648 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.676238059997559 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.675326347351074 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.673474311828613 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.676054954528809 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.676846504211426 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.677567481994629 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.67350959777832 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.674263000488281 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.674751281738281 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.676502227783203 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.675909996032715 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.676013946533203 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.677160263061523 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.676111221313477 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.675052642822266 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.67632007598877 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.676314353942871 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.676329612731934 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.676634788513184 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.677085876464844 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.676518440246582 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.674410820007324 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.67582893371582 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.67530345916748 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.677226066589355 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.674527168273926 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.677281379699707 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.676190376281738 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.67458438873291 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.676701545715332 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.674578666687012 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.677021980285645 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.675920486450195 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.676774978637695 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.675968170166016 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.675092697143555 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.676567077636719 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.675466537475586 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.676915168762207 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.676345825195312 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.675810813903809 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.680309295654297 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.6746244430542 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.675800323486328 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.673392295837402 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.676492691040039 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.67574691772461 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.67611312866211 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.676393508911133 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.67338752746582 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.675232887268066 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.672069549560547 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.676436424255371 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.676228523254395 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.675576210021973 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.678200721740723 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.672150611877441 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.673097610473633 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.670710563659668 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.677223205566406 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.675731658935547 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.675774574279785 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.676533699035645 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.672196388244629 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.675130844116211 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.672260284423828 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.676352500915527 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.67531681060791 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.675251007080078 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.67506217956543 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.672934532165527 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.672183990478516 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.67196273803711 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.677040100097656 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.67768383026123 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.675640106201172 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.679840087890625 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.673463821411133 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.669708251953125 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.671504974365234 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.676878929138184 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.676836013793945 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.675243377685547 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.677521705627441 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.660104751586914 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.666590690612793 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.663055419921875 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.442667961120605 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.444175720214844 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.443778038024902 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.440760612487793 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.44139289855957 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.442167282104492 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.441547393798828 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.4439697265625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.443235397338867 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.44385814666748 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.443076133728027 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.443568229675293 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.44163703918457 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.442699432373047 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.443207740783691 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.443011283874512 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.442501068115234 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.44402027130127 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.441267967224121 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.444022178649902 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.440688133239746 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.443564414978027 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.442070007324219 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.44461441040039 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.441737174987793 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.441076278686523 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.444948196411133 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.441740989685059 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.443011283874512 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.442789077758789 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.441352844238281 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.441828727722168 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.44018268585205 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.442389488220215 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.443504333496094 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.444395065307617 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.443867683410645 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.443160057067871 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.443187713623047 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.438167572021484 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.441858291625977 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.440536499023438 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.443629264831543 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.443604469299316 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.442819595336914 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.442479133605957 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.441357612609863 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.441352844238281 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.440287590026855 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.444184303283691 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.44356632232666 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.442991256713867 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.444071769714355 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.440995216369629 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.442502975463867 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.444186210632324 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.443278312683105 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.443392753601074 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.444107055664062 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.440227508544922 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.44158935546875 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.444012641906738 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.442598342895508 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.444076538085938 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.443713188171387 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.443499565124512 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.442349433898926 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.441582679748535 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.443160057067871 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.443716049194336 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.442867279052734 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.443025588989258 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.443700790405273 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.446552276611328 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.442344665527344 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.4426851272583 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.44405460357666 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.443264961242676 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.443314552307129 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.443138122558594 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.441915512084961 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.442842483520508 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.442408561706543 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.442349433898926 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.44381046295166 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.443429946899414 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.44421672821045 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.44393539428711 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.443354606628418 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.443439483642578 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.444048881530762 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.442757606506348 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.443404197692871 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.442607879638672 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.441681861877441 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.441627502441406 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.442399024963379 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.442588806152344 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.443036079406738 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.442946434020996 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.444198608398438 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.44216251373291 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.443787574768066 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.443464279174805 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.445137977600098 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.443564414978027 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.442192077636719 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.44461441040039 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.443306922912598 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.443967819213867 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.442705154418945 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.442976951599121 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.443459510803223 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.443655967712402 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.444535255432129 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.445446968078613 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.443489074707031 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.443005561828613 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.443591117858887 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.443283081054688 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.443267822265625 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.444465637207031 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.443890571594238 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.444104194641113 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.442665100097656 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.442876815795898 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.443671226501465 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.443573951721191 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.443914413452148 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.444151878356934 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.443058967590332 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.442760467529297 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.442747116088867 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.444037437438965 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.443416595458984 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.442771911621094 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.443593978881836 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.442296981811523 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.442480087280273 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.443371772766113 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.443604469299316 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.444074630737305 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.443434715270996 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.442802429199219 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.443357467651367 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.443416595458984 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.440210342407227 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.444085121154785 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.443803787231445 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.443076133728027 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.443132400512695 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.44211196899414 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.442218780517578 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.443513870239258 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.443655967712402 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.443093299865723 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.443258285522461 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.442742347717285 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.44183349609375 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.440402030944824 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.44167423248291 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.4431734085083 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.442428588867188 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.443385124206543 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.443155288696289 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.442461013793945 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.442896842956543 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.44083309173584 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.443609237670898 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.443379402160645 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.442662239074707 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.443798065185547 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.442328453063965 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.44071102142334 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.441168785095215 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.443349838256836 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.444764137268066 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.442145347595215 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.444584846496582 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.440247535705566 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.441627502441406 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.438939094543457 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.443320274353027 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.443357467651367 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.443387031555176 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.444435119628906 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.440255165100098 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.441023826599121 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.44118595123291 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.443329811096191 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.44332504272461 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.44328498840332 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.445371627807617 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.433932304382324 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.433636665344238 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.436285972595215 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.317365646362305 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.3170804977417 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.315113067626953 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.316485404968262 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.317566871643066 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.317193984985352 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.316753387451172 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.317429542541504 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.31772232055664 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.317630767822266 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.317286491394043 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.315831184387207 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.317183494567871 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.317538261413574 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.317666053771973 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.319061279296875 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.317885398864746 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.316733360290527 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.315624237060547 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.317848205566406 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.318243980407715 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.317793846130371 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.316180229187012 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.317112922668457 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.319324493408203 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.317779541015625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.316474914550781 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.31709098815918 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.317477226257324 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.318207740783691 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.317533493041992 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.31690788269043 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.317292213439941 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.318745613098145 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.31567668914795 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.317708015441895 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.318659782409668 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.317055702209473 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.316893577575684 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.316622734069824 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.316563606262207 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.319725036621094 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.31889820098877 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.318089485168457 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.3182954788208 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.31832504272461 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.315890312194824 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.31530475616455 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.318611145019531 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.317521095275879 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.31735610961914 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.317222595214844 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.315566062927246 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.315840721130371 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.317646980285645 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.316062927246094 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.317882537841797 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.31795597076416 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.316957473754883 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.318022727966309 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.314935684204102 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.315262794494629 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.315518379211426 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.317658424377441 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.317502975463867 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.317813873291016 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.317164421081543 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.317800521850586 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.316060066223145 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.316571235656738 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.318058967590332 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.317429542541504 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.318258285522461 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.314087867736816 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.315796852111816 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.317230224609375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.316328048706055 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.31758975982666 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.318217277526855 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.31786060333252 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.317331314086914 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.316967010498047 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.316024780273438 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.315730094909668 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.31806468963623 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.318138122558594 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.318977355957031 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.316494941711426 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.317304611206055 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.316577911376953 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.317169189453125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.31808090209961 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.31832218170166 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.318307876586914 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.316746711730957 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.315695762634277 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.316283226013184 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.315719604492188 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.318147659301758 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.317513465881348 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.317705154418945 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.320101737976074 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.319439888000488 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.317719459533691 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.318055152893066 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.317449569702148 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.317341804504395 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.316362380981445 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.317253112792969 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.31749439239502 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.316229820251465 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.31772518157959 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.317927360534668 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.317971229553223 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.318187713623047 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.31570053100586 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.317341804504395 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.316352844238281 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.317414283752441 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.318187713623047 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.318153381347656 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.318192481994629 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.316490173339844 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.317496299743652 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.31812572479248 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.317825317382812 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.318002700805664 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.317459106445312 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.31836223602295 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.316244125366211 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.317474365234375 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.317144393920898 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.317329406738281 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.318182945251465 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.31694507598877 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.318408966064453 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.318197250366211 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.316658020019531 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.315673828125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.31677532196045 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.318105697631836 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.31755542755127 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.318039894104004 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.317002296447754 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.316544532775879 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.316225051879883 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.317831039428711 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.317835807800293 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.317527770996094 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.318079948425293 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.31860065460205 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.31701374053955 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.317764282226562 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.316150665283203 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.317577362060547 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.317710876464844 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.317758560180664 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.317675590515137 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.317619323730469 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.318448066711426 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.315346717834473 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.317459106445312 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.318155288696289 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.317371368408203 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.31914234161377 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.315624237060547 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.315651893615723 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.315954208374023 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.317832946777344 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.317158699035645 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.318055152893066 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.318005561828613 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.316258430480957 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.31711483001709 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.31611156463623 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.317672729492188 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.317630767822266 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.317276954650879 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.316947937011719 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.315230369567871 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.31496524810791 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.314921379089355 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.317538261413574 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.317633628845215 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.31742000579834 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.319570541381836 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.316052436828613 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.316691398620605 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.31684398651123 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.317815780639648 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.317774772644043 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.317429542541504 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.31742000579834 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.310721397399902 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.312508583068848 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.312742233276367 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.213951110839844 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.213761329650879 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.216739654541016 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.211606979370117 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.21341323852539 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.213969230651855 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.213164329528809 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.214395523071289 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.213720321655273 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.213849067687988 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.213082313537598 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.214737892150879 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.214932441711426 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.212729454040527 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.214231491088867 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.21406078338623 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.214421272277832 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.213342666625977 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.213257789611816 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.213922500610352 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.21264362335205 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.214348793029785 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.213433265686035 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.21391487121582 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.213057518005371 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.211932182312012 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.214153289794922 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.21359634399414 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.213364601135254 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.214436531066895 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.213072776794434 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.213208198547363 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.212361335754395 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.212300300598145 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.214153289794922 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.214387893676758 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.213603019714355 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.214983940124512 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.213890075683594 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.212504386901855 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.213471412658691 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.209765434265137 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.213452339172363 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.213817596435547 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.214051246643066 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.213228225708008 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.213744163513184 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.214043617248535 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.211981773376465 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.214202880859375 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.21406078338623 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.213933944702148 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.214494705200195 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.212721824645996 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.212726593017578 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.213179588317871 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.21370792388916 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.214151382446289 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.214792251586914 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.213155746459961 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.213457107543945 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.214986801147461 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.214543342590332 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.214367866516113 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.21358871459961 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.21347713470459 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.21450138092041 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.212875366210938 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.214035987854004 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.21346664428711 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.213812828063965 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.213783264160156 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.213568687438965 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.21700382232666 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.213177680969238 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.213257789611816 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.215493202209473 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.213749885559082 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.214140892028809 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.214338302612305 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.213455200195312 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.213521003723145 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.213350296020508 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.213571548461914 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.214217185974121 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.214635848999023 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.21362018585205 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.21429443359375 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.213642120361328 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.213496208190918 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.21414852142334 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.213725090026855 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.214071273803711 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.213593482971191 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.212994575500488 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.213433265686035 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.21341323852539 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.213905334472656 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.214151382446289 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.213715553283691 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.214115142822266 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.212843894958496 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.21362018585205 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.212544441223145 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.213496208190918 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.213642120361328 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.214140892028809 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.215787887573242 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.21568775177002 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.214309692382812 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.214046478271484 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.21401309967041 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.214167594909668 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.213822364807129 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.214178085327148 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.216797828674316 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.21412181854248 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.213715553283691 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.21450424194336 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.213977813720703 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.213690757751465 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.214344024658203 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.21536636352539 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.21386432647705 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.213590621948242 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.213690757751465 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.213715553283691 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.214057922363281 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.21364688873291 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.215076446533203 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.214057922363281 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.214007377624512 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.214311599731445 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.214374542236328 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.213846206665039 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.214651107788086 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.213963508605957 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.213586807250977 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.213808059692383 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.214095115661621 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.213929176330566 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.213839530944824 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.213808059692383 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.213933944702148 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.213846206665039 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.214178085327148 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.212376594543457 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.214373588562012 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.214221954345703 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.214051246643066 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.214217185974121 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.213802337646484 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.213491439819336 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.213583946228027 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.214076042175293 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.214016914367676 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.213706016540527 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.213676452636719 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.2133207321167 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.21264934539795 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.213094711303711 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.214162826538086 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.213802337646484 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.214005470275879 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.214118957519531 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.213593482971191 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.21373462677002 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.212762832641602 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.214117050170898 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.21435832977295 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.213485717773438 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.213630676269531 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.21264934539795 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.213554382324219 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.212656021118164 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.213910102844238 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.214126586914062 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.21385669708252 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.214200019836426 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.21375846862793 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.21373462677002 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.21338176727295 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.21374225616455 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.213827133178711 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.213359832763672 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.214611053466797 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.212215423583984 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.2135591506958 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.212570190429688 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.213678359985352 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.213868141174316 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.213793754577637 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.214316368103027 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.208498001098633 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.209223747253418 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.209026336669922 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.159443855285645 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.159516334533691 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.159503936767578 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.160867691040039 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.160138130187988 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.159215927124023 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.159904479980469 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.159976959228516 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.160082817077637 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.160306930541992 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.159223556518555 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.15882396697998 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.159478187561035 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.158843040466309 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.159744262695312 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.15987491607666 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.159734725952148 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.158673286437988 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.158978462219238 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.16012191772461 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.160518646240234 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.16006851196289 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.159574508666992 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.160447120666504 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.15939998626709 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.159942626953125 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.160526275634766 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.159462928771973 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.15990161895752 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.159981727600098 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.159482955932617 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.159584999084473 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.159601211547852 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.160233497619629 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.159700393676758 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.160030364990234 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.16003704071045 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.160165786743164 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.159996032714844 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.159584999084473 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.160181999206543 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.163278579711914 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.160126686096191 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.160181999206543 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.16018009185791 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.160543441772461 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.157787322998047 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.159002304077148 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.160529136657715 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.159964561462402 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.16012954711914 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.16020393371582 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.159725189208984 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.158232688903809 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.16085433959961 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.159884452819824 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.160082817077637 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.160285949707031 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.159063339233398 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.160917282104492 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.158402442932129 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.159138679504395 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.158926010131836 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.159860610961914 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.160266876220703 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.159937858581543 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.159823417663574 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.159814834594727 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.159193992614746 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.161362648010254 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.160005569458008 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.159782409667969 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.160364151000977 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.15740966796875 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.160640716552734 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.159710884094238 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.159346580505371 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.159823417663574 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.16012191772461 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.159560203552246 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.160306930541992 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.160096168518066 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.159133911132812 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.161136627197266 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.160429954528809 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.16012191772461 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.16037368774414 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.159971237182617 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.160025596618652 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.161294937133789 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.157772064208984 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.160117149353027 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.16015625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.16006088256836 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.158160209655762 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.158331871032715 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.159371376037598 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.159148216247559 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.160107612609863 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.160348892211914 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.16049575805664 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.160524368286133 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.160826683044434 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.159915924072266 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.16154956817627 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.159865379333496 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.159821510314941 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.16028881072998 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.157276153564453 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.159993171691895 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.159945487976074 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.159801483154297 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.15976619720459 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.159865379333496 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.160097122192383 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.15900993347168 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.160225868225098 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.159300804138184 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.159690856933594 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.160011291503906 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.160244941711426 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.160192489624023 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.159380912780762 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.159879684448242 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.158867835998535 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.160022735595703 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.160131454467773 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.159662246704102 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.160558700561523 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.159102439880371 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.160075187683105 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.159753799438477 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.15918254852295 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.159967422485352 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.160307884216309 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.159797668457031 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.160555839538574 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.160082817077637 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.160332679748535 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.159337043762207 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.160126686096191 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.160258293151855 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.160223007202148 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.159675598144531 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.159651756286621 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.159618377685547 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.160615921020508 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.16015625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.159845352172852 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.160097122192383 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.160592079162598 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.159804344177246 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.160558700561523 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.159385681152344 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.159778594970703 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.160005569458008 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.159843444824219 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.159841537475586 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.15971565246582 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.160238265991211 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.159189224243164 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.160063743591309 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.159926414489746 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.159725189208984 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.159380912780762 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.159157752990723 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.159201622009277 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.158857345581055 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.160000801086426 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.159693717956543 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.160151481628418 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.160019874572754 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.159177780151367 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.158770561218262 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.158663749694824 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.160238265991211 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.159729957580566 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.159245491027832 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.159934997558594 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.15785026550293 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.15863037109375 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.15773868560791 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.160019874572754 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.160088539123535 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.15979290008545 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.160773277282715 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.159184455871582 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.158855438232422 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.159404754638672 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.160126686096191 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.15981674194336 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.159521102905273 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.160529136657715 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.156387329101562 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.156983375549316 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.158339500427246 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.114678382873535 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.11498737335205 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.11557388305664 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.113097190856934 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.115793228149414 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.11535930633545 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.11493968963623 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.115370750427246 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.115321159362793 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.114891052246094 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.114230155944824 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.115670204162598 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.115952491760254 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.115057945251465 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.11525821685791 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.115119934082031 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.115768432617188 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.115797996520996 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.115706443786621 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.114888191223145 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.114531517028809 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.115021705627441 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.114688873291016 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.115242958068848 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.115720748901367 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.115070343017578 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.115602493286133 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.114669799804688 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.114949226379395 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.115696907043457 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.115137100219727 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.114550590515137 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.11379337310791 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.115957260131836 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.116581916809082 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.11525821685791 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.115097999572754 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.115547180175781 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.114818572998047 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.11240291595459 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.113762855529785 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.111863136291504 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.115118026733398 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.115209579467773 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.115580558776855 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.115226745605469 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.115239143371582 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.115348815917969 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.114912986755371 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.115245819091797 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.115004539489746 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.115236282348633 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.115070343017578 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.11464786529541 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.114225387573242 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.114992141723633 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.115100860595703 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.11535930633545 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.116206169128418 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.11377239227295 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.115071296691895 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.11562442779541 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.115446090698242 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.115653038024902 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.11485767364502 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.11501693725586 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.115130424499512 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.114977836608887 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.115002632141113 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.113746643066406 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.115070343017578 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.11530590057373 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.115286827087402 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.115774154663086 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.11483383178711 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.1151762008667 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.115108489990234 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.11543083190918 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.115111351013184 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.115422248840332 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.113521575927734 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.114842414855957 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.114678382873535 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.115253448486328 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.11543083190918 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.115411758422852 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.114958763122559 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.114283561706543 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.115660667419434 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.114770889282227 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.116944313049316 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.115198135375977 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.115242958068848 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.114842414855957 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.115463256835938 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.114239692687988 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.113398551940918 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.114567756652832 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.115419387817383 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.115286827087402 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.115057945251465 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.1145658493042 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.11538314819336 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.116000175476074 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.114861488342285 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.114891052246094 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.11485767364502 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.115585327148438 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.115937232971191 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.115071296691895 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.11401081085205 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.115448951721191 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.115591049194336 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.115580558776855 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.115364074707031 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.116422653198242 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.116277694702148 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.115411758422852 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.115288734436035 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.115476608276367 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.115226745605469 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.115452766418457 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.116097450256348 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.115653038024902 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.115310668945312 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.114269256591797 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.115127563476562 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.115217208862305 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.114628791809082 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.115281105041504 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.115171432495117 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.114706039428711 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.115909576416016 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.115368843078613 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.11512279510498 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.115850448608398 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.115561485290527 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.115137100219727 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.11483097076416 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.115103721618652 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.115351676940918 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.115159034729004 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.115460395812988 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.115528106689453 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.115113258361816 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.115198135375977 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.113919258117676 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.11522102355957 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.114992141723633 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.115078926086426 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.114684104919434 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.115280151367188 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.114272117614746 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.114958763122559 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.11522388458252 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.115026473999023 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.1151762008667 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.115498542785645 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.115084648132324 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.11492919921875 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.115078926086426 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.115537643432617 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.11546802520752 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.115233421325684 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.115509033203125 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.115002632141113 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.11538314819336 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.114534378051758 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.11523151397705 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.11532974243164 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.114760398864746 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.114768028259277 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.114775657653809 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.115050315856934 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.114840507507324 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.115021705627441 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.115334510803223 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.115446090698242 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.115240097045898 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.114949226379395 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.114744186401367 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.114659309387207 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.115050315856934 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.11498737335205 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.115078926086426 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.115821838378906 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.114676475524902 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.114129066467285 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.113984107971191 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.11535930633545 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.114893913269043 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.115152359008789 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.115740776062012 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.110855102539062 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.110512733459473 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.111279487609863 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.074488639831543 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.074613571166992 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.074769973754883 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.074707984924316 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.074085235595703 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.074605941772461 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.075084686279297 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.074618339538574 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.075101852416992 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.074952125549316 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.07343864440918 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.073912620544434 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.07400131225586 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.073982238769531 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.074686050415039 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.075163841247559 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.074203491210938 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.073705673217773 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.073734283447266 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.075350761413574 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.076225280761719 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.074909210205078 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.073895454406738 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.074774742126465 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.074624061584473 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.074627876281738 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.074434280395508 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.074524879455566 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.074461936950684 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.074477195739746 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.074207305908203 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.073734283447266 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.074207305908203 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.073333740234375 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.073847770690918 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.07474136352539 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.074817657470703 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.074397087097168 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.074431419372559 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.075535774230957 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.073775291442871 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.076271057128906 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.074442863464355 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.074882507324219 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.074332237243652 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.074479103088379 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.073107719421387 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.073724746704102 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.074385643005371 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.0750150680542 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.07443618774414 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.074814796447754 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.074029922485352 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.073472023010254 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.075451850891113 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.074409484863281 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.074918746948242 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.074912071228027 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.074328422546387 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.075329780578613 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.073533058166504 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.073862075805664 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.073436737060547 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.074251174926758 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.074806213378906 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.074466705322266 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.074798583984375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.074116706848145 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.073967933654785 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.076528549194336 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.074931144714355 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.074645042419434 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.074697494506836 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.073182106018066 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.074164390563965 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.074132919311523 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.074769973754883 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.074589729309082 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.074461936950684 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.074188232421875 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.075404167175293 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.074978828430176 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.074481010437012 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.074746131896973 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.075056076049805 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.074822425842285 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.074813842773438 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.075576782226562 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.074399948120117 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.074385643005371 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.072660446166992 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.07463550567627 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.074543952941895 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.074395179748535 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.073175430297852 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.074841499328613 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.074836730957031 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.0737943649292 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.07450008392334 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.074673652648926 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.07524585723877 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.075370788574219 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.074860572814941 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.073281288146973 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.075807571411133 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.074567794799805 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.074453353881836 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.074995994567871 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.074630737304688 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.075178146362305 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.074337005615234 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.073760986328125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.074426651000977 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.074906349182129 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.074803352355957 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.074399948120117 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.074188232421875 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.07400131225586 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.074145317077637 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.07474136352539 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.07474136352539 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.074974060058594 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.074039459228516 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.075096130371094 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.074399948120117 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.074719429016113 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.074976921081543 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.074605941772461 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.074798583984375 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.07425594329834 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.074901580810547 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.074702262878418 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.073847770690918 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.074711799621582 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.074716567993164 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.074352264404297 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.074539184570312 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.074317932128906 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.074515342712402 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.074524879455566 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.074841499328613 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.074649810791016 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.074959754943848 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.074846267700195 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.07473373413086 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.074515342712402 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.074700355529785 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.074957847595215 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.074771881103516 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.07443618774414 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.075356483459473 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.074092864990234 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.075118064880371 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.074090003967285 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.074645042419434 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.074481010437012 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.074610710144043 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.074455261230469 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.074322700500488 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.07502269744873 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.07427978515625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.074668884277344 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.074458122253418 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.07486629486084 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.074268341064453 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.074399948120117 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.073786735534668 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.07357406616211 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.074649810791016 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.074621200561523 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.0746431350708 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.074904441833496 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.074735641479492 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.07352066040039 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.07412338256836 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.074678421020508 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.074735641479492 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.074515342712402 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.07473373413086 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.073994636535645 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.073631286621094 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.072929382324219 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.074707984924316 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.074707984924316 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.074597358703613 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.0748929977417 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.073929786682129 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.074572563171387 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.074678421020508 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.074390411376953 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.074613571166992 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.074548721313477 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.074966430664062 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.072348594665527 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.073851585388184 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.073240280151367 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.037837028503418 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.037497520446777 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.038172721862793 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.037765502929688 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.038607597351074 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.037389755249023 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.03767204284668 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.037751197814941 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.037851333618164 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.03796672821045 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.035980224609375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.038126945495605 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.038559913635254 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.038686752319336 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.037540435791016 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.037736892700195 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.037796974182129 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.03801441192627 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.037736892700195 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.037981033325195 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.037425994873047 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.038307189941406 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.037239074707031 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.037713050842285 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.037755966186523 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.037473678588867 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.03882884979248 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.038784980773926 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.037693977355957 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.038105010986328 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.038002014160156 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.0375337600708 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.037487983703613 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.038326263427734 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.038483619689941 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.038315773010254 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.037832260131836 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.038098335266113 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.037765502929688 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.03662109375 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.037353515625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.036392211914062 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.037985801696777 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.038077354431152 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.03810977935791 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.038043022155762 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.03737735748291 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.038421630859375 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.036918640136719 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.037894248962402 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.037885665893555 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.037453651428223 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.03769588470459 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.037691116333008 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.036685943603516 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.037851333618164 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.038227081298828 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.03782844543457 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.037947654724121 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.037471771240234 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.037161827087402 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.037511825561523 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.038047790527344 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.038115501403809 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.037602424621582 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.038165092468262 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.037291526794434 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.036972999572754 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.038637161254883 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.036497116088867 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.038139343261719 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.037920951843262 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.03791332244873 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.038204193115234 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.037674903869629 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.0376558303833 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.037878036499023 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.038162231445312 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.038026809692383 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.038066864013672 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.037220001220703 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.038277626037598 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.0366792678833 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.037758827209473 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.037801742553711 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.03806209564209 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.037935256958008 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.037487983703613 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.037296295166016 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.037334442138672 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.038459777832031 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.038066864013672 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.037995338439941 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.03797435760498 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.03823184967041 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.03723430633545 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.037243843078613 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.038268089294434 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.03774642944336 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.0376558303833 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.037732124328613 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.03738021850586 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.038191795349121 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.039052963256836 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.037981033325195 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.037724494934082 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.038239479064941 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.037848472595215 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.038201332092285 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.03791618347168 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.037928581237793 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.03803825378418 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.037982940673828 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.037593841552734 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.037954330444336 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.038829803466797 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.038223266601562 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.03808879852295 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.038559913635254 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.037693977355957 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.03789234161377 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.038385391235352 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.038268089294434 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.037931442260742 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.037464141845703 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.037829399108887 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.0376558303833 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.038033485412598 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.037866592407227 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.038328170776367 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.037617683410645 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.037418365478516 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.03862190246582 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.03793716430664 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.037602424621582 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.038540840148926 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.038153648376465 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.038212776184082 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.036703109741211 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.037593841552734 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.038077354431152 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.037841796875 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.03828239440918 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.038077354431152 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.03791618347168 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.03791618347168 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.03750228881836 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.037823677062988 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.037724494934082 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.037879943847656 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.037856101989746 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.038250923156738 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.037193298339844 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.038023948669434 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.038115501403809 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.037931442260742 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.037760734558105 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.037636756896973 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.038153648376465 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.037952423095703 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.037487983703613 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.037969589233398 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.03782844543457 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.037689208984375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.038021087646484 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.037702560424805 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.037736892700195 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.037717819213867 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.037848472595215 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.037813186645508 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.037806510925293 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.037440299987793 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.036966323852539 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.038594245910645 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.036928176879883 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.03799057006836 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.037674903869629 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.037705421447754 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.037717819213867 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.037348747253418 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.03812026977539 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.037789344787598 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.037617683410645 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.037808418273926 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.037875175476074 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.038077354431152 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.036784172058105 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.037372589111328 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.037145614624023 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.038139343261719 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.03786849975586 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.037640571594238 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.038837432861328 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.035650253295898 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.034812927246094 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.03567886352539 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.015263557434082 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.015190124511719 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.015426635742188 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.015303611755371 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.014955520629883 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.015273094177246 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.0151948928833 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.01530647277832 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.015146255493164 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.015743255615234 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.01552677154541 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.014817237854004 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.014325141906738 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.015327453613281 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.015433311462402 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.01538372039795 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.015168190002441 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.01517105102539 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.01501750946045 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.014979362487793 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.015968322753906 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.015165328979492 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.014626502990723 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.015384674072266 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.015079498291016 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.015252113342285 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.014726638793945 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.01480484008789 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.014936447143555 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.015020370483398 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.01506519317627 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.014545440673828 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.015233039855957 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.014796257019043 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.014850616455078 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.01578140258789 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.01483154296875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.015399932861328 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.015314102172852 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.016201972961426 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.01551628112793 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.015071868896484 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.015182495117188 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.015022277832031 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.015244483947754 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.015098571777344 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.014955520629883 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.014191627502441 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.014907836914062 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.01492691040039 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.015399932861328 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.015318870544434 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.015079498291016 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.014678001403809 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.016549110412598 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.01490306854248 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.014941215515137 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.01518726348877 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.01502513885498 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.015098571777344 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.01455020904541 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.015690803527832 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.014812469482422 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.015233039855957 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.01487922668457 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.015101432800293 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.016185760498047 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.015519142150879 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.01453971862793 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.016302108764648 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.015203475952148 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.015146255493164 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.015483856201172 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.014901161193848 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.015103340148926 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.015252113342285 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.01489543914795 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.014793395996094 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.015446662902832 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.015361785888672 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.015376091003418 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.015089988708496 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.014914512634277 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.015810012817383 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.015342712402344 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.014967918395996 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.015339851379395 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.015655517578125 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.01516342163086 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.015337944030762 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.014488220214844 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.015151977539062 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.01504135131836 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.014688491821289 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.01480770111084 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.01489543914795 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.015165328979492 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.01407241821289 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.015376091003418 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.015079498291016 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.015519142150879 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.015989303588867 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.01566219329834 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.014296531677246 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.015528678894043 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.015342712402344 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.014984130859375 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.015480995178223 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.014979362487793 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.015727043151855 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.014683723449707 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.014626502990723 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.015314102172852 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.015179634094238 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.015485763549805 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.014742851257324 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.015292167663574 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.01483154296875 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.014697074890137 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.01506519317627 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.01518726348877 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.015337944030762 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.015103340148926 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.015155792236328 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.014817237854004 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.014955520629883 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.015514373779297 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.015237808227539 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.015007972717285 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.014554023742676 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.015588760375977 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.01490306854248 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.014657020568848 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.015342712402344 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.015325546264648 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.014564514160156 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.015411376953125 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.01488208770752 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.015027046203613 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.014534950256348 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.015244483947754 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.015299797058105 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.015571594238281 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.014925003051758 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.01489543914795 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.014577865600586 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.014604568481445 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.015146255493164 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.015155792236328 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.015134811401367 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.01516056060791 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.015271186828613 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.015349388122559 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.014480590820312 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.014798164367676 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.01480484008789 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.01538372039795 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.015230178833008 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.014912605285645 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.0148286819458 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.01502513885498 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.01499080657959 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.015244483947754 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.015260696411133 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.015103340148926 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.014914512634277 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.014410972595215 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.01432991027832 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.015314102172852 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.014971733093262 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.01501750946045 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.015220642089844 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.015168190002441 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.014021873474121 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.01499080657959 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.014782905578613 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.0155668258667 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.015299797058105 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.014809608459473 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.01516342163086 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.014196395874023 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.014545440673828 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.01534652709961 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.01504135131836 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.015314102172852 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.015058517456055 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.015222549438477 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.015060424804688 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.014955520629883 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.015084266662598 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.015395164489746 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.015146255493164 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.015643119812012 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.013763427734375 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.013914108276367 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.013551712036133 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 9.991853713989258 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 9.991497039794922 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 9.991786003112793 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 9.992063522338867 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 9.991840362548828 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 9.992063522338867 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 9.992478370666504 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 9.991342544555664 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 9.991802215576172 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 9.99177360534668 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 9.989343643188477 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 9.991896629333496 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 9.992478370666504 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 9.9921875 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 9.991708755493164 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 9.99179744720459 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 9.991837501525879 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 9.991842269897461 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 9.992039680480957 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 9.991783142089844 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 9.991183280944824 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 9.991918563842773 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 9.991430282592773 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 9.991148948669434 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 9.99170207977295 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 9.991687774658203 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 9.991859436035156 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 9.991928100585938 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 9.99184513092041 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 9.992378234863281 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 9.991939544677734 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 9.991907119750977 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 9.991008758544922 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 9.99181079864502 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 9.992833137512207 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 9.991840362548828 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 9.991721153259277 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 9.991996765136719 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 9.991945266723633 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 9.990937232971191 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 9.990982055664062 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 9.99046516418457 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 9.992035865783691 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 9.992044448852539 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 9.991878509521484 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 9.992192268371582 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 9.991172790527344 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 9.992650032043457 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 9.99230670928955 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 9.992268562316895 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 9.9916353225708 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 9.99170207977295 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 9.991859436035156 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 9.991716384887695 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 9.990920066833496 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 9.992217063903809 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 9.992116928100586 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 9.991764068603516 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 9.99203109741211 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 9.99207592010498 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 9.991268157958984 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 9.99142074584961 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 9.991275787353516 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 9.99223518371582 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 9.991826057434082 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 9.99184513092041 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 9.991172790527344 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 9.991533279418945 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 9.991816520690918 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 9.9908447265625 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 9.991678237915039 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 9.992087364196777 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 9.991971015930176 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 9.99221420288086 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 9.992042541503906 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 9.992419242858887 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 9.992709159851074 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 9.992268562316895 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 9.992128372192383 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 9.992077827453613 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 9.992120742797852 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 9.991851806640625 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 9.991659164428711 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 9.990784645080566 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 9.99179458618164 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 9.99186897277832 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 9.992042541503906 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 9.99133014678955 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 9.991622924804688 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 9.99203109741211 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 9.991732597351074 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 9.992106437683105 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 9.992217063903809 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 9.991859436035156 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 9.99156379699707 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 9.99118709564209 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 9.991016387939453 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 9.991382598876953 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 9.992112159729004 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 9.991692543029785 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 9.991945266723633 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 9.99097728729248 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 9.992411613464355 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 9.991988182067871 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 9.992793083190918 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 9.991853713989258 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 9.991958618164062 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 9.991740226745605 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 9.99128246307373 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 9.992297172546387 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 9.991775512695312 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 9.992101669311523 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 9.991930961608887 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 9.991888046264648 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 9.991928100585938 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 9.992283821105957 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 9.9921875 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 9.9918851852417 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 9.992573738098145 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 9.991878509521484 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 9.991842269897461 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 9.99163818359375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 9.992416381835938 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 9.99181842803955 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 9.992230415344238 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 9.992159843444824 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 9.991654396057129 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 9.992087364196777 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 9.992199897766113 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 9.9927339553833 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 9.991934776306152 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 9.992147445678711 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 9.992630958557129 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 9.992015838623047 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 9.992009162902832 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 9.992206573486328 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 9.992144584655762 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 9.992080688476562 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 9.990987777709961 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 9.992058753967285 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 9.991915702819824 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 9.99203109741211 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 9.992240905761719 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 9.991654396057129 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 9.991783142089844 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 9.99222183227539 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 9.991921424865723 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 9.992063522338867 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 9.991921424865723 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 9.991896629333496 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 9.991975784301758 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 9.992283821105957 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 9.991435050964355 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 9.991559028625488 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 9.992159843444824 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 9.992247581481934 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 9.9920072555542 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 9.99223518371582 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 9.991992950439453 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 9.992273330688477 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 9.991662979125977 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 9.992020606994629 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 9.991930961608887 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 9.992112159729004 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 9.991926193237305 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 9.992077827453613 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 9.9923095703125 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 9.991737365722656 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 9.99181842803955 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 9.99209976196289 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 9.991910934448242 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 9.99153995513916 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 9.991697311401367 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 9.992085456848145 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 9.991140365600586 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 9.992074012756348 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 9.991950035095215 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 9.991775512695312 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 9.992178916931152 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 9.99125862121582 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 9.992454528808594 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 9.99123764038086 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 9.991673469543457 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 9.99202823638916 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 9.991668701171875 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 9.99179744720459 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 9.990954399108887 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 9.991511344909668 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 9.991479873657227 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 9.992050170898438 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 9.9920015335083 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 9.992063522338867 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 9.992464065551758 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 9.990560531616211 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 9.989849090576172 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 9.991116523742676 |
| } |
| ] |
| } |
| ] |
| } |
