torchao-testing
/

opt-125m-ModuleFqnToConfig-v1-regex-0.14.0.dev

Model card Files Files and versions

jerryzh168 commited on Oct 3

Commit

3d3afd3

·

verified ·

1 Parent(s): 4820374

Update README.md

Files changed (1) hide show

README.md +7 -4

README.md CHANGED Viewed

@@ -53,11 +53,14 @@ qconfig_dict = {
     # vllm
     "model.decoder.layers.3.self_attn.qkv_proj": int4wo,
-    "model.decoder.layers.*.self_attn.q_proj": float8dyn,
-    "model.decoder.layers.*.self_attn.k_proj": float8dyn,
-    "model.decoder.layers.*.self_attn.v_proj": float8dyn,
     # vllm
-    "model.decoder.layers.*.self_attn.qkv_proj": float8dyn,
     "_default": intxwo,
 }

     # vllm
     "model.decoder.layers.3.self_attn.qkv_proj": int4wo,
+    "re:model\.decoder\.layers\..+\.self_attn\.q_proj": float8dyn,
+    "re:model\.decoder\.layers\..+\.self_attn\.k_proj": float8dyn,
+    "re:model\.decoder\.layers\..+\.self_attn\.v_proj": float8dyn,
+    # this should not take effect and we'll fallback to _default
+    # since no full mach (missing `j` in the end)
+    "re:model\.decoder\.layers\..+\.self_attn\.out_pro": float8dyn,
     # vllm
+    "re:model\.decoder\.layers\..+\.self_attn\.qkv_proj": float8dyn,
     "_default": intxwo,
 }