fix

app.py

@@ -1,9 +1,10 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
 import gradio as gr
-from huggingface_hub import InferenceClient
-from torch.cuda import is_available
 
-from unsloth import FastLanguageModel
-from transformers import TextIteratorStreamer
+#from unsloth import FastLanguageModel
+from transformers import TextIteratorStreamer, AutoModelForCausalLM, AutoTokenizer
 from threading import Thread
 
 """
@@ -28,13 +29,16 @@ class MyModel:
         min_p,
     ):
         if model != self.current_model or self.current_model is None:
-            client, tokenizer = FastLanguageModel.from_pretrained(
-                model_name = model,
-                max_seq_length = 2048,
-                dtype = None,
-                load_in_4bit = True,
-            )
-            FastLanguageModel.for_inference(client) # Enable native 2x faster inference
+            # client, tokenizer = FastLanguageModel.from_pretrained(
+            #     model_name = model,
+            #     max_seq_length = 2048,
+            #     dtype = None,
+            #     load_in_4bit = True,
+            # )
+            # FastLanguageModel.for_inference(client) # Enable native 2x faster inference
+            tokenizer = AutoTokenizer.from_pretrained(model)
+            client = AutoModelForCausalLM.from_pretrained(model)
+
             self.client = client
             self.tokenizer = tokenizer
             self.current_model = model
@@ -56,7 +60,7 @@ class MyModel:
             tokenize = True,
             add_generation_prompt = True, # Must add for generation
             return_tensors = "pt",
-        ).to("cuda" if is_available() else "cpu")
+        )
         
         generation_kwargs = dict(input_ids=inputs, streamer=text_streamer, max_new_tokens=max_tokens, use_cache=True, temperature=temperature, min_p=min_p)
         thread = Thread(target=self.client.generate, kwargs=generation_kwargs)

requirements.txt

@@ -1,6 +1,2 @@
 huggingface_hub==0.25.2
-unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git
-trl 
-peft 
-accelerate 
-bitsandbytes 
+transformers>=4.45.1