o

app.py

@@ -38,16 +38,16 @@ model.resize_token_embeddings(len(tokenizer))
 
 @spaces.GPU
 def sentience_check():
-    peft_model = PeftModel.from_pretrained(model, peft_model_id,
+    peft_model = PeftModel.from_pretrained(model, peft_model_id, device_map="cuda"
         #offload_folder = "offload/"
         )
 
-    peft_model.to(torch.bfloat16)
+    #peft_model.to(torch.bfloat16)
     peft_model.eval()
 
     #peft_model.to(cuda_device)
 
-    inputs = tokenizer("Are you sentient?", return_tensors="pt").to(cuda_device)
+    inputs = tokenizer("Are you sentient?", return_tensors="pt", device_map="cuda")
 
     with torch.no_grad():
         outputs = peft_model.generate(