Changes to bit and bytes

app.py

@@ -16,6 +16,7 @@ model = AutoModelForCausalLM.from_pretrained(
     device_map="auto",
     trust_remote_code=True,
     torch_dtype=torch.float16,  # Use float16 for memory efficiency
+    low_cpu_mem_usage=True,  # Add this for better memory handling
 )
 tokenizer = AutoTokenizer.from_pretrained("./fine-tuned-model")
 tokenizer.pad_token = tokenizer.eos_token
@@ -28,6 +29,7 @@ base_model = AutoModelForCausalLM.from_pretrained(
     device_map="auto",
     trust_remote_code=True,
     torch_dtype=torch.float16,
+    low_cpu_mem_usage=True,  # Add this for better memory handling
 )
 
 def generate_response(

requirements.txt

@@ -1,12 +1,14 @@
 torch>=2.0.0
-transformers>=4.30.0
-datasets>=2.12.0
-accelerate>=0.20.0
-bitsandbytes>=0.41.0
-peft>=0.4.0
-pytorch-lightning>=2.0.0
-gradio>=3.40.0
-wandb>=0.15.0
-rich>=13.0.0
+transformers>=4.36.0
+datasets>=2.14.0
+accelerate>=0.25.0
+peft>=0.7.0
+trl>=0.7.4
+wandb>=0.16.0
+gradio>=4.0.0
+rich>=13.7.0
+numpy>=1.24.0
+tqdm>=4.66.0
+scipy>=1.11.0
 sentencepiece>=0.1.99
 protobuf>=4.23.0