Add quantization

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 from huggingface_hub import login
 from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextIteratorStreamer
 from threading import Thread
+import torch
 
 MODEL = "m-a-p/OpenCodeInterpreter-DS-33B"
 
@@ -19,8 +20,15 @@ def messages_for(python):
         {"role": "user", "content": user_prompt_for(python)}
     ]
 
+quant_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_quant_type="nf4"
+)
+
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
-model = AutoModelForCausalLM.from_pretrained(MODEL)
+model = AutoModelForCausalLM.from_pretrained(MODEL, device_map="auto", quantization_config=quant_config)
 streamer = TextIteratorStreamer(tokenizer)
 
 cplusplus = None