jerryzh168
/

gemma-3-12b-it-INT4

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on Oct 11

Commit

202b498

·

verified ·

1 Parent(s): 17a4220

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -135,7 +135,7 @@ model_to_quantize = "google/gemma-3-12b-it"
 from torchao.quantization import Int4WeightOnlyConfig
-quant_config = Int4WeightOnlyConfig(group_size=128, int4_packing_format="tile_packed_to_4d", int4_choose_qparams_algorithm="hqq")
 quantization_config = TorchAoConfig(quant_type=quant_config)
 quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="cuda:0", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
 tokenizer = AutoTokenizer.from_pretrained(model_id)

 from torchao.quantization import Int4WeightOnlyConfig
+quant_config = Int4WeightOnlyConfig(group_size=128)
 quantization_config = TorchAoConfig(quant_type=quant_config)
 quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="cuda:0", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
 tokenizer = AutoTokenizer.from_pretrained(model_id)