Spaces:

UcsTurkey
/

mistral7b

Paused

App Files Files Community

ciyidogan commited on May 20

Commit

01fdd15

verified ·

1 Parent(s): 65269fa

Update fine_tune_inference_test_mistral.py

Browse files

Files changed (1) hide show

fine_tune_inference_test_mistral.py +28 -47

fine_tune_inference_test_mistral.py CHANGED Viewed

@@ -1,19 +1,15 @@
-import os, torch, zipfile, threading, uvicorn, time, traceback
 from fastapi import FastAPI
 from fastapi.responses import HTMLResponse, JSONResponse
 from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import PeftModel
-from huggingface_hub import hf_hub_download
 from datetime import datetime
 import random
 # === Sabitler ===
 HF_TOKEN = os.environ.get("HF_TOKEN")
 MODEL_BASE = "mistralai/Mistral-7B-Instruct-v0.2"
-FINE_TUNE_ZIP = "trained_model_000_009.zip"
-FINE_TUNE_REPO = "UcsTurkey/trained-zips"
-USE_FINE_TUNE = False   # ✅ Ana modeli test etmek için False yap
 USE_SAMPLING = False
 CONFIDENCE_THRESHOLD = -1.5
 FALLBACK_ANSWERS = [
@@ -47,7 +43,7 @@ def root():
     <html>
     <body>
         <h2>Mistral 7B Chat</h2>
-        <textarea id=\"input\" rows=\"4\" cols=\"60\" placeholder=\"SORU: ...\"></textarea><br>
         <button onclick=\"send()\">Gönder</button>
         <pre id=\"output\"></pre>
         <script>
@@ -77,7 +73,8 @@ def chat(msg: Message):
         if not user_input:
             return {"error": "Boş giriş"}
-        prompt = f"SORU: {user_input}\nCEVAP:"
         inputs = tokenizer(prompt, return_tensors="pt")
         if not inputs or "input_ids" not in inputs:
@@ -86,18 +83,23 @@ def chat(msg: Message):
         inputs = inputs.to(model.device)
         with torch.no_grad():
-            output = model.generate(
-                **inputs,
-                max_new_tokens=128,
-                do_sample=USE_SAMPLING,
-                temperature=0.7 if USE_SAMPLING else None,
-                top_p=0.9 if USE_SAMPLING else None,
-                top_k=50 if USE_SAMPLING else None,
-                return_dict_in_generate=True,
-                output_scores=True,
-                suppress_tokens=[tokenizer.pad_token_id]
-            )
         decoded = tokenizer.decode(output.sequences[0], skip_special_tokens=True)
         answer = decoded[len(prompt):].strip()
@@ -130,37 +132,16 @@ def setup_model():
     global model, tokenizer
     try:
         device, supports_bf16 = detect_env()
-        dtype = torch.bfloat16 if supports_bf16 else torch.float32
         log(f"🧠 Ortam: {device.upper()}, dtype: {dtype}")
-        if USE_FINE_TUNE:
-            log("📦 Fine-tune zip indiriliyor...")
-            zip_path = hf_hub_download(
-                repo_id=FINE_TUNE_REPO,
-                filename=FINE_TUNE_ZIP,
-                repo_type="model",
-                token=HF_TOKEN
-            )
-            extract_path = "/app/extracted"
-            os.makedirs(extract_path, exist_ok=True)
-            with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                zip_ref.extractall(extract_path)
-            tokenizer = AutoTokenizer.from_pretrained(os.path.join(extract_path, "output"))
-            if tokenizer.pad_token is None:
-                tokenizer.pad_token = tokenizer.eos_token
-            base = AutoModelForCausalLM.from_pretrained(MODEL_BASE, torch_dtype=dtype).to(device)
-            peft = PeftModel.from_pretrained(base, os.path.join(extract_path, "output"))
-            model = peft.model.to(device)
-        else:
-            log("🧪 Sadece ana model yüklenecek...")
-            tokenizer = AutoTokenizer.from_pretrained(MODEL_BASE, use_fast=False)
-            if tokenizer.pad_token is None:
-                tokenizer.pad_token = tokenizer.eos_token
-            model = AutoModelForCausalLM.from_pretrained(MODEL_BASE, torch_dtype=dtype).to(device)
         model.eval()
         log("✅ Model başarıyla yüklendi.")

+import os, torch, threading, uvicorn, time, traceback
 from fastapi import FastAPI
 from fastapi.responses import HTMLResponse, JSONResponse
 from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM
 from datetime import datetime
 import random
 # === Sabitler ===
 HF_TOKEN = os.environ.get("HF_TOKEN")
 MODEL_BASE = "mistralai/Mistral-7B-Instruct-v0.2"
+USE_FINE_TUNE = False
 USE_SAMPLING = False
 CONFIDENCE_THRESHOLD = -1.5
 FALLBACK_ANSWERS = [
     <html>
     <body>
         <h2>Mistral 7B Chat</h2>
+        <textarea id=\"input\" rows=\"4\" cols=\"60\" placeholder=\"Write your instruction...\"></textarea><br>
         <button onclick=\"send()\">Gönder</button>
         <pre id=\"output\"></pre>
         <script>
         if not user_input:
             return {"error": "Boş giriş"}
+        # ✅ Ana modelin beklediği instruct formatı
+        prompt = f"### Instruction:\n{user_input}\n\n### Response:"
         inputs = tokenizer(prompt, return_tensors="pt")
         if not inputs or "input_ids" not in inputs:
         inputs = inputs.to(model.device)
+        generate_args = {
+            "max_new_tokens": 128,
+            "return_dict_in_generate": True,
+            "output_scores": True,
+            "suppress_tokens": [tokenizer.pad_token_id],
+            "do_sample": USE_SAMPLING
+        }
+        if USE_SAMPLING:
+            generate_args.update({
+                "temperature": 0.7,
+                "top_p": 0.9,
+                "top_k": 50
+            })
         with torch.no_grad():
+            output = model.generate(**inputs, **generate_args)
         decoded = tokenizer.decode(output.sequences[0], skip_special_tokens=True)
         answer = decoded[len(prompt):].strip()
     global model, tokenizer
     try:
         device, supports_bf16 = detect_env()
+        dtype = torch.float32  # daha kararlı
         log(f"🧠 Ortam: {device.upper()}, dtype: {dtype}")
+        log("🧪 Sadece ana model yüklenecek...")
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_BASE, use_fast=False)
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+        model = AutoModelForCausalLM.from_pretrained(MODEL_BASE, torch_dtype=dtype).to(device)
         model.eval()
         log("✅ Model başarıyla yüklendi.")