Update app.py

app.py

@@ -99,8 +99,8 @@ def clean_visible_text(text: str) -> str:
     text = re.sub(r"https?://\S+|www\.\S+|%[0-9A-Fa-f]{2}", " ", text)  
     return text.strip()  
   
-START_CUES = ("الحمد لله", "أما بعد", "فالجواب", "الجواب", "الإجابة")  
-END_CUES   = ("والله أعلم", "والله تعالى أعلم")  
+START_CUES = ("الحمد لله","أما بعد","فالجواب","الجواب","الإجابة","الإجابــة")
+END_CUES   = ("والله أعلم","والله تعالى أعلم","وبالله التوفيق")
   
 def slice_to_answer_core(text: str) -> str:  
     start_idx = min([text.find(p) for p in START_CUES if p in text] or [0])  
@@ -120,7 +120,199 @@ def dynamic_snippet(text: str) -> str:
   
 def looks_religious_answer(text: str) -> bool:  
     return any(k in text for k in START_CUES + END_CUES)  
-  
+  # ===================== Google CSE (نسخة Colab المحسّنة) =====================
+HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; HajeenBot/1.0)"}
+
+def normalize_for_score(s: str) -> str:
+    # نعيد استخدام نفس منطق التطبيع تبعك
+    return normalize_text(s or "")
+
+def call_google_cse(query, cx=CUSTOM_SEARCH_ENGINE_ID, key=GOOGLE_API_KEY, num=10, lr="lang_ar"):
+    if key in (None, "", "YOUR_API_KEY") or cx in (None, "", "YOUR_CSE_ID"):
+        raise RuntimeError("Google CSE keys are not configured")
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {"q": query, "cx": cx, "key": key, "num": num, "lr": lr, "safe": "off"}
+    resp = requests.get(url, params=params, timeout=25, headers=HEADERS)
+    resp.raise_for_status()
+    return resp.json().get("items", []) or []
+
+def get_soup(url, timeout=20):
+    try:
+        resp = requests.get(url, headers=HEADERS, timeout=timeout)
+        resp.encoding = resp.apparent_encoding or "utf-8"
+        resp.raise_for_status()
+        soup = BeautifulSoup(resp.text, "html.parser")
+        for tag in soup(["script","style","noscript","header","footer","nav","form","aside"]):
+            tag.extract()
+        return soup
+    except Exception:
+        return None
+
+def page_text(soup: BeautifulSoup) -> str:
+    text = soup.get_text(" ", strip=True)
+    text = re.sub(r"\u0640", "", text)  # إزالة التطويل
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"https?://\S+|www\.\S+|%[0-9A-Fa-f]{2}", " ", text)
+    return text.strip()
+
+# --- قواطع سؤال/جواب حسب الدومين ---
+def slice_by_cues(text: str) -> str:
+    low = text
+    start = 0
+    for cue in START_CUES:
+        i = low.find(cue)
+        if i != -1:
+            start = i
+            break
+    end = len(text)
+    for cue in END_CUES:
+        j = low.find(cue, start)
+        if j != -1:
+            end = j
+            break
+    core = text[start:end].strip()
+    return core if len(core.split()) > 15 else text.strip()
+
+def extract_islamweb_qa(text: str):
+    m = re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الإجاب+ة|الإجابة|الجواب)\s*[:：]?", text, flags=re.DOTALL)
+    q = (m.group(1).strip() if m else "")
+    ans_region = text[m.end():].strip() if m else ""
+    answer = slice_by_cues(ans_region or text)
+    return q, answer
+
+def extract_islamqa_qa(text: str):
+    m = re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة)\s*[:：]?", text, flags=re.DOTALL)
+    q = (m.group(1).strip() if m else "")
+    ans_region = text[m.end():].strip() if m else ""
+    answer = slice_by_cues(ans_region or text)
+    return q, answer
+
+def extract_binbaz_qa(text: str):
+    m = re.search(r"(?:س|السؤال)\s*[:：]?\s*(.+?)\s*(?:ج|الجواب)\s*[:：]?", text, flags=re.DOTALL)
+    q = (m.group(1).strip() if m else "")
+    ans_region = text[m.end():].strip() if m else ""
+    answer = slice_by_cues(ans_region or text)
+    return q, answer
+
+def extract_alifta_qa(text: str):
+    m = re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة)\s*[:：]?", text, flags=re.DOTALL)
+    q = (m.group(1).strip() if m else "")
+    ans_region = text[m.end():].strip() if m else ""
+    answer = slice_by_cues(ans_region or text)
+    return q, answer
+
+def extract_generic_qa(text: str):
+    m = re.search(r"(?:السؤال|س)\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة|ج)\s*[:：]?", text, flags=re.DOTALL)
+    q = (m.group(1).strip() if m else "")
+    ans_region = text[m.end():].strip() if m else ""
+    answer = slice_by_cues(ans_region or text)
+    return q, answer
+
+DOMAIN_EXTRACTORS = {
+    "islamweb.net": extract_islamweb_qa,
+    "islamqa.info": extract_islamqa_qa,
+    "binbaz.org.sa": extract_binbaz_qa,
+    "alifta.gov.sa": extract_alifta_qa,
+}
+
+def choose_extractor(url):
+    host = urlparse(url).netloc.lower()
+    for dom, fn in DOMAIN_EXTRACTORS.items():
+        if dom in host:
+            return fn
+    return extract_generic_qa
+
+def structural_score(url: str, page_text_sample: str) -> int:
+    u = url.lower()
+    score = 0
+    if re.search(r"/fatwa/\d+", u) or "/answers/" in u or "/fatwas/" in u:
+        score += 30
+    if "/fatawa/" in u or "العرض الموضوعي" in page_text_sample or "بحث عن فتوى" in page_text_sample:
+        score -= 35
+    if page_text_sample.count(".. المزيد") > 1 or re.search(r"\b\d+\s+\d+\s+\d+\b", page_text_sample):
+        score -= 15
+    return score
+
+def score_candidate(user_query, cand_title, cand_url, page_question, page_answer, snippet):
+    nq_user = normalize_for_score(user_query)
+    nq_page_q = normalize_for_score(page_question)
+    nq_title  = normalize_for_score(cand_title or "")
+    nq_snip   = normalize_for_score(snippet or "")
+    wc = len((page_answer or "").split())
+
+    s_q = fuzz.token_set_ratio(nq_user, nq_page_q) if nq_page_q else 0
+    s_t = fuzz.partial_ratio(nq_user, nq_title) if nq_title else 0
+    s_s = fuzz.partial_ratio(nq_user, nq_snip)  if nq_snip else 0
+
+    len_adj = 0
+    if wc < 40:
+        len_adj -= 15
+    elif wc > 5000:
+        len_adj -= 10
+
+    host = urlparse(cand_url).netloc.lower()
+    domain_bonus = 8 if any(d in host for d in ("islamweb.net","islamqa.info","binbaz.org.sa","alifta.gov.sa")) else 0
+
+    sample = (page_question + " " + page_answer)[:1200]
+    struct = structural_score(cand_url, sample)
+
+    score = int(0.80*s_q + 0.15*s_t + 0.05*s_s + len_adj + domain_bonus + struct)
+    return score, {"s_q": s_q, "s_title": s_t, "s_snip": s_s, "wc": wc, "struct": struct}
+
+def google_search_match_and_extract_full(
+    user_query: str,
+    domains=("islamweb.net","islamqa.info","binbaz.org.sa","alifta.gov.sa"),
+    num_results=10,
+    max_links=8,
+    sleep_between=0.8
+):
+    site_filter = " OR ".join(f"site:{d}" for d in domains)
+    query = f"{user_query} ({site_filter})"
+
+    items = call_google_cse(query, num=num_results)
+    if not items:
+        return {"query": user_query, "best": None, "candidates": []}
+
+    candidates = []
+    for item in items[:max_links]:
+        title   = item.get("title","")
+        link    = item.get("link","")
+        snippet = item.get("snippet","")
+
+        soup = get_soup(link)
+        if not soup:
+            page_q, page_a = "", snippet
+        else:
+            txt = page_text(soup)
+            extractor = choose_extractor(link)
+            try:
+                page_q, page_a = extractor(txt)
+            except Exception:
+                page_q, page_a = extract_generic_qa(txt)
+
+        score, meta = score_candidate(user_query, title, link, page_q, page_a, snippet)
+        entry = {
+            "title": title,
+            "url": link,
+            "snippet": snippet,
+            "page_question": page_q,
+            "answer": page_a,
+            "score": score,
+            **meta
+        }
+
+        # قصّ خاص لابن باز: أول 250 كلمة فقط
+        if "binbaz.org.sa" in urlparse(link).netloc.lower():
+            words = (entry["answer"] or "").split()
+            if len(words) > 250:
+                entry["answer"] = " ".join(words[:250]) + " …"
+
+        candidates.append(entry)
+        time.sleep(sleep_between)
+
+    candidates_sorted = sorted(candidates, key=lambda x: x["score"], reverse=True)
+    best = candidates_sorted[0] if candidates_sorted else None
+    return {"query": user_query, "best": best, "candidates": candidates_sorted}
 # ===================== 5) Load assets & FAISS (فتاوى) =====================  
 print("[SERVER-INFO] بدء تحميل الأصول...")  
 DATA_FILE_ID = "1GMG6fVxhUuBEAHP91c8RAUdUJh5TxY5O"  
@@ -175,92 +367,41 @@ def save_feedback(question: str, answer: str, useful: str, comment: str = ""):
         df_learned.to_csv(learned_data_path, index=False)  
     return row  
     
-# ===================== 8) Google Search (النسخة النهائية القوية) =====================
-    
-# --- أدوات مساعدة خاصة بالبحث الجديد ---
-def choose_extractor(url):
-    host = urlparse(url).netloc.lower()
-    DOMAIN_EXTRACTORS = {
-        "islamweb.net": lambda text: (re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الإجاب+ة|الإجابة|الجواب)", text, re.DOTALL), text),
-        "islamqa.info": lambda text: (re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة)", text, re.DOTALL), text),
-        "binbaz.org.sa": lambda text: (re.search(r"(?:س|السؤال)\s*[:：]?\s*(.+?)\s*(?:ج|الجواب)", text, re.DOTALL), text),
-        "alifta.gov.sa": lambda text: (re.search(r"السؤال\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة)", text, re.DOTALL), text),
-    }
-    for dom, func in DOMAIN_EXTRACTORS.items():
-        if dom in host:
-            return func
-    # الدالة الافتراضية
-    return lambda text: (re.search(r"(?:السؤال|س)\s*[:：]?\s*(.+?)\s*(?:الجواب|الإجابة|ج)", text, re.DOTALL), text)
-
-def get_page_text(url):
-    try:
-        resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
-        resp.raise_for_status()
-        resp.encoding = resp.apparent_encoding or "utf-8"
-        soup = BeautifulSoup(resp.text, "html.parser")
-        for tag in soup(["script","style","noscript","header","footer","nav","form","aside"]):
-            tag.extract()
-        return clean_visible_text(soup.get_text(" ", strip=True))
-    except Exception:
-        return ""
-
-# --- الدالة الرئيسية للبحث ---
 def google_search_fatwa(query: str):
-    if GOOGLE_API_KEY == "YOUR_API_KEY" or CUSTOM_SEARCH_ENGINE_ID == "YOUR_CSE_ID":
-        print("[GOOGLE-SEARCH] مفاتيح جوجل غير مضبوطة.")
-        return None
+    """
+    يغلّف النسخة المحسّنة: يبحث بالمواقع الموثوقة، يستخرج سؤال/جواب ويعيد أفضل نتيجة.
+    يحفظ التعلم الذاتي في learned_fatwas.csv مثل قبل.
+    """
     try:
-        service = build("customsearch", "v1", developerKey=GOOGLE_API_KEY)
-        site_filter = " OR ".join(f"site:{d}" for d in ("islamweb.net","islamqa.info","binbaz.org.sa","alifta.gov.sa"))
-        full_query = f'{query} ({site_filter})'
-        res = service.cse().list(q=full_query, cx=CUSTOM_SEARCH_ENGINE_ID, num=8, lr="lang_ar", safe="off").execute()
-        items = res.get("items", [])
-        if not items: return None
-
-        candidates = []
-        for item in items:
-            url = item.get("link", "")
-            title = item.get("title", "")
-                
-            page_content = get_page_text(url)
-            if not page_content: continue
-
-            extractor = choose_extractor(url)
-            match, text_region = extractor(page_content)
-                
-            page_question = match.group(1).strip() if match else ""
-            answer_region = text_region[match.end():].strip() if match else text_region
-            final_answer = slice_to_answer_core(answer_region) # نستخدم دالة القص هنا فقط على منطقة الجواب
-
-            if len(final_answer.split()) < 40: continue # تجاهل الإجابات القصيرة جدًا
-
-            # حساب درجة التشابه
-            score = fuzz.token_set_ratio(normalize_text(query), normalize_text(page_question)) if page_question else 0
-                
-            candidates.append({
-                "title": title,
-                "url": url,
-                "answer": final_answer,
-                "score": score
-            })
-
-        if not candidates: return None
-            
-        best = max(candidates, key=lambda c: c["score"])
-            
-        if best['score'] < 50: # نرفع الحد الأدنى للثقة
-            print(f"[GOOGLE-SEARCH] أفضل نتيجة درجتها ضعيفة ({best['score']}). يتم تجاهلها.")
+        res = google_search_match_and_extract_full(query)
+        best = res.get("best")
+        if not best:
             return None
 
-        # لا داعي لحفظ النتيجة في التعلم الذاتي هنا، لنركز على الدقة أولاً
-            
+        answer_text = best["answer"] or best["snippet"] or ""
+        title = best.get("title","")
+        url   = best.get("url","")
+        wc    = len(answer_text.split())
+        score = int(best.get("score", 70))
+
+        # خزّن تعلم ذاتي
+        new_row = pd.DataFrame([{
+            "question": query,
+            "answer": answer_text,
+            "source_url": url,
+            "title": title,
+            "word_count": wc,
+            "score": score
+        }])
+        new_row.to_csv(learned_data_path, mode='a', header=not os.path.exists(learned_data_path), index=False)
+
         return {
             "question": query,
-            "answer": best["answer"], # الجواب المقصوص والنظيف
+            "answer": answer_text,
             "source": "بحث جوجل (فتوى موثقة)",
-            "source_url": best["url"],
-            "title": best["title"],
-            "score": best['score']
+            "source_url": url,
+            "title": title,
+            "score": score
         }
     except Exception as e:
         print(f"[GOOGLE-SEARCH-ERROR] {e}")
@@ -290,6 +431,18 @@ def safe_download(file_id, output_path):
             print(f"[SAFE-DOWNLOAD-INFO] الملف {output_path} موجود بالفعل، سيتم استخدامه.")
         else:
             raise e
+# --- Google Drive IDs & local paths (أحاديث) ---  
+# --- Google Drive IDs & local paths (أحاديث) ---  
+ID_BUKHARI = os.environ.get("ID_BUKHARI")  
+ID_MUSLIM  = os.environ.get("ID_MUSLIM")  
+ID_MUSNAD  = os.environ.get("ID_MUSNAD")  
+  
+PATHS = {  
+    "bukhari": os.path.join(DATA_DIR, "sahih_bukhari_clean.csv"),
+    "muslim":  os.path.join(DATA_DIR, "sahih_muslim_clean.csv"),
+    "musnad":  os.path.join(DATA_DIR, "musnad_ahmed_clean.csv"),
+}
+  
 @app.on_event("startup")  
 async def startup_event():  
     global df_main, df_learned, question_embeddings, index, tokenizer, model  
@@ -400,18 +553,7 @@ def feedback(req: FeedbackRequest):
   
 # ===================== 11) ----- قسم ا��أحاديث (مدمج) ----- =====================  
   
-# --- Google Drive IDs & local paths (أحاديث) ---  
-# --- Google Drive IDs & local paths (أحاديث) ---  
-ID_BUKHARI = os.environ.get("ID_BUKHARI")  
-ID_MUSLIM  = os.environ.get("ID_MUSLIM")  
-ID_MUSNAD  = os.environ.get("ID_MUSNAD")  
-  
-PATHS = {  
-    "bukhari": os.path.join(DATA_DIR, "sahih_bukhari_clean.csv"),
-    "muslim":  os.path.join(DATA_DIR, "sahih_muslim_clean.csv"),
-    "musnad":  os.path.join(DATA_DIR, "musnad_ahmed_clean.csv"),
-}
-  
+
 # --- تطبيع عربي (أحاديث) ---  
 def normalize_ar(s: str) -> str:  
     if not isinstance(s, str):