t

app.py

@@ -4,13 +4,39 @@ import os
 import spaces
 import torch
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, Qwen2_5_VLForConditionalGeneration
+from datasets import load_dataset
 
 huggingface_hub.login(os.getenv('HF_TOKEN'))
-tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
-model = AutoModelForCausalLM.from_pretrained("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
-cuda_device = torch.device("cuda")
-cpu_device = torch.device("cpu")
+peft_model_id = "debisoft/Qwen2.5-VL-3B-Instruct-thinking-function_calling-V0"
+
+bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16,
+            bnb_4bit_use_double_quant=True,
+        )
+
+device = "auto"
+config = PeftConfig.from_pretrained(peft_model_id)
+model = Qwen2_5_VLForConditionalGeneration.from_pretrained(config.base_model_name_or_path,
+        #AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
+                                             quantization_config=bnb_config,
+                                             device_map="auto",
+                                             )
+tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
+model.resize_token_embeddings(len(tokenizer))
+model = PeftModel.from_pretrained(model, peft_model_id,
+    #offload_folder = "offload/"
+    )
+
+model.to(torch.bfloat16)
+model.eval()
+
+#tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
+#model = AutoModelForCausalLM.from_pretrained("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
+
 
 @spaces.GPU
 def sentience_check():