Upload backend/chatbot/chatbot.py with huggingface_hub

backend/chatbot/chatbot.py

@@ -0,0 +1,882 @@
+"""
+Chatbot with ML-based intent classification for natural language queries.
+"""
+import json
+import re
+import unicodedata
+from pathlib import Path
+from typing import Dict, List, Tuple, Any, Optional
+import joblib
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.naive_bayes import MultinomialNB
+from sklearn.pipeline import Pipeline
+import numpy as np
+from hue_portal.core.models import Procedure, Fine, Office, Advisory
+from hue_portal.core.search_ml import search_with_ml, expand_query_with_synonyms
+
+
+def format_fine_amount(min_fine: Optional[float], max_fine: Optional[float]) -> Optional[str]:
+    """
+    Format fine amount as '200.000 - 400.000 VNĐ'.
+    
+    Args:
+        min_fine: Minimum fine amount.
+        max_fine: Maximum fine amount.
+    
+    Returns:
+        Formatted string or None if both are None.
+    """
+    if min_fine is not None and max_fine is not None:
+        # Format with thousand separators (dots for Vietnamese format)
+        min_str = f"{min_fine:,.0f}".replace(",", ".")
+        max_str = f"{max_fine:,.0f}".replace(",", ".")
+        return f"{min_str} - {max_str} VNĐ"
+    elif min_fine is not None:
+        min_str = f"{min_fine:,.0f}".replace(",", ".")
+        return f"{min_str} VNĐ"
+    return None
+
+
+# Training data for intent classification (fallback when chưa có model huấn luyện)
+INTENT_TRAINING_DATA = {
+    "search_fine": [
+        "mức phạt", "phạt bao nhiêu", "tiền phạt", "vi phạm giao thông",
+        "vượt đèn đỏ", "nồng độ cồn", "không đội mũ bảo hiểm",
+        "mức phạt là gì", "phạt như thế nào", "hành vi vi phạm",
+        "điều luật", "nghị định", "mức xử phạt"
+    ],
+    "search_procedure": [
+        "thủ tục", "làm thủ tục", "hồ sơ", "điều kiện",
+        "thủ tục cư trú", "thủ tục ANTT", "thủ tục PCCC",
+        "cần giấy tờ gì", "làm như thế nào", "quy trình",
+        "thời hạn", "lệ phí", "nơi nộp"
+    ],
+    "search_office": [
+        "địa chỉ", "điểm tiếp dân", "công an", "phòng ban",
+        "số điện thoại", "giờ làm việc", "nơi tiếp nhận",
+        "đơn vị nào", "ở đâu", "liên hệ"
+    ],
+    "search_advisory": [
+        "cảnh báo", "lừa đảo", "scam", "thủ đoạn",
+        "cảnh giác", "an toàn", "bảo mật",
+        "cảnh báo lừa đảo giả danh công an",
+        "mạo danh cán bộ công an",
+        "lừa đảo mạo danh",
+        "cảnh báo an ninh",
+        "thủ đoạn lừa đảo",
+        "scam giả danh",
+        "cảnh giác lừa đảo online",
+        "lừa đảo qua điện thoại",
+        "cảnh báo bảo mật",
+        "mạo danh cán bộ",
+        "lừa đảo giả danh",
+        "cảnh báo lừa đảo",
+        "thủ đoạn scam",
+        "cảnh giác an toàn",
+        "lừa đảo online",
+        "cảnh báo mạo danh"
+    ],
+    "search_legal": [
+        "quyết định", "quy định", "thông tư", "nghị quyết",
+        "văn bản pháp luật", "văn bản quy phạm", "điều lệnh",
+        "kỷ luật đảng viên", "kỷ luật", "xử lý kỷ luật",
+        "quyết định 69", "quyết định 264", "qd 69", "qd 264",
+        "thông tư 02", "tt 02", "điều lệnh cand",
+        "quy định kỷ luật", "hình thức kỷ luật", "mức kỷ luật",
+        "xử lý vi phạm", "kỷ luật đảng", "kỷ luật cán bộ",
+        "quy định về", "theo quyết định", "theo thông tư",
+        "nội dung quyết định", "nội dung thông tư", "điều khoản"
+    ],
+    "general_query": [
+        "xin chào", "giúp tôi", "tư vấn", "hỏi",
+        "thông tin", "tra cứu", "tìm kiếm"
+    ]
+}
+
+# Đường dẫn artefact model
+TRAINING_DIR = Path(__file__).resolve().parent / "training"
+ARTIFACT_MODEL = TRAINING_DIR / "artifacts" / "intent_model.joblib"
+ARTIFACT_METRICS = TRAINING_DIR / "artifacts" / "metrics.json"
+
+# Legal-related keywords reused across intent calibration
+LEGAL_KEYWORDS = [
+    "quyết định",
+    "quy định",
+    "thông tư",
+    "nghị quyết",
+    "văn bản pháp luật",
+    "văn bản quy phạm",
+    "điều lệnh",
+    "kỷ luật đảng viên",
+    "kỷ luật",
+    "xử lý kỷ luật",
+    "hình thức kỷ luật",
+    "mức kỷ luật",
+    "quyết định 69",
+    "quyết định 264",
+    "qd 69",
+    "qd 264",
+    "thông tư 02",
+    "tt 02",
+]
+
+# Response templates
+RESPONSE_TEMPLATES = {
+    "search_fine": "Tôi tìm thấy {count} mức phạt liên quan đến '{query}':",
+    "search_procedure": "Tôi tìm thấy {count} thủ tục liên quan đến '{query}':",
+    "search_office": "Tôi tìm thấy {count} đơn vị liên quan đến '{query}':",
+    "search_advisory": "Tôi tìm thấy {count} cảnh báo liên quan đến '{query}':",
+    "general_query": "Tôi có thể giúp bạn tra cứu thông tin về thủ tục, mức phạt, đơn vị hoặc cảnh báo. Bạn muốn tìm gì?",
+    "no_results": "Xin lỗi, tôi không tìm thấy thông tin liên quan đến '{query}'. Vui lòng thử lại với từ khóa khác.",
+    "greeting": "Xin chào! Tôi có thể giúp bạn tra cứu các thông tin liên quan về các văn bản quy định pháp luật về xử lý kỷ luật cán bộ, đảng viên.",
+}
+
+
+class Chatbot:
+    def __init__(self):
+        self.intent_classifier = None
+        self.intent_metrics: Optional[Dict[str, Any]] = None
+        self._load_classifier()
+    
+    def _load_classifier(self):
+        """Load pretrained classifier nếu có, fallback tự huấn luyện seed data."""
+        if ARTIFACT_MODEL.exists():
+            try:
+                self.intent_classifier = joblib.load(ARTIFACT_MODEL)
+                if ARTIFACT_METRICS.exists():
+                    self.intent_metrics = json.loads(ARTIFACT_METRICS.read_text(encoding="utf-8"))
+                return
+            except Exception as exc:
+                print(f"Warning: không thể load intent_model.joblib ({exc}). Sẽ huấn luyện tạm thời.")
+                self.intent_classifier = None
+                self.intent_metrics = None
+        self._train_classifier()
+    
+    def _train_classifier(self):
+        """Train intent classification model."""
+        try:
+            # Prepare training data
+            texts = []
+            labels = []
+            
+            for intent, examples in INTENT_TRAINING_DATA.items():
+                for example in examples:
+                    texts.append(self._preprocess_text(example))
+                    labels.append(intent)
+            
+            if not texts:
+                return
+            
+            # Create and train pipeline
+            self.intent_classifier = Pipeline([
+                ('tfidf', TfidfVectorizer(
+                    analyzer='word',
+                    ngram_range=(1, 2),
+                    min_df=1,
+                    lowercase=True,
+                    token_pattern=r'\b\w+\b'
+                )),
+                ('clf', MultinomialNB())
+            ])
+            
+            self.intent_classifier.fit(texts, labels)
+        except Exception as e:
+            print(f"Error training classifier: {e}")
+            self.intent_classifier = None
+    
+    def _preprocess_text(self, text: str) -> str:
+        """Preprocess text for classification - keep Vietnamese characters."""
+        if not text:
+            return ""
+        text = text.lower().strip()
+        # Only remove punctuation marks, keep all letters (including Vietnamese) and numbers
+        # Remove: !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~
+        text = re.sub(r'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]', ' ', text)
+        text = re.sub(r'\s+', ' ', text)
+        return text.strip()
+
+    def _remove_accents(self, text: str) -> str:
+        """Remove diacritics for accent-insensitive matching."""
+        if not text:
+            return ""
+        normalized = unicodedata.normalize("NFD", text)
+        return "".join(ch for ch in normalized if unicodedata.category(ch) != "Mn")
+
+    def _keyword_in(self, query_lower: str, query_ascii: str, keyword: str) -> bool:
+        """Check keyword presence in either original or accent-free text."""
+        kw_lower = keyword.lower()
+        if kw_lower in query_lower:
+            return True
+        kw_ascii = self._remove_accents(kw_lower)
+        return kw_ascii in query_ascii
+    
+    def classify_intent(self, query: str, context: Optional[Dict[str, Any]] = None) -> Tuple[str, float]:
+        """
+        Classify user intent from query with optional context.
+        
+        Args:
+            query: User query string.
+            context: Optional context dictionary with recent_messages, entities, etc.
+        
+        Returns:
+            Tuple of (intent, confidence_score)
+        """
+        # Use context to improve classification if available
+        if context:
+            # Check recent intents in context
+            recent_messages = context.get("recent_messages", [])
+            if recent_messages:
+                # Get most recent intent
+                for msg in reversed(recent_messages):
+                    if msg.get("intent") and msg.get("intent") != "greeting":
+                        recent_intent = msg.get("intent")
+                        # Boost confidence if query is short and recent intent is relevant
+                        if len(query.split()) <= 5:
+                            # Likely a follow-up question
+                            return (recent_intent, 0.85)
+        
+        model_intent, model_confidence = self._model_based_intent(query)
+        keyword_intent, keyword_confidence = self._keyword_based_intent(query)
+        
+        chosen_intent = keyword_intent
+        confidence = keyword_confidence
+
+        # Nếu model tự tin và không mâu thuẫn với keyword, ưu tiên model
+        if model_intent and model_confidence >= 0.65:
+            if keyword_intent in {model_intent, "general_query", "greeting"}:
+                chosen_intent = model_intent
+                confidence = max(confidence, model_confidence)
+        
+        # Ensemble: combine model and keyword predictions
+        if model_intent and keyword_intent:
+            if model_intent == keyword_intent:
+                # Both agree - boost confidence
+                confidence = min(1.0, (model_confidence + keyword_confidence) / 2 + 0.1)
+            elif model_confidence > 0.7 and keyword_confidence < 0.6:
+                # Model is more confident - use model
+                chosen_intent = model_intent
+                confidence = model_confidence * 0.9  # Slight penalty for disagreement
+
+        # Special handling for greeting - only if really simple
+        if keyword_intent == "greeting":
+            query_lower = query.lower().strip()
+            query_ascii = self._remove_accents(query_lower)
+            query_words = query_lower.split()
+            # Double-check: if query has fine keywords, it's NOT a greeting
+            fine_indicators = ["phạt", "mức", "vuot", "vượt", "đèn", "den", "vi phạm", "vi pham"]
+            if any(self._keyword_in(query_lower, query_ascii, indicator) for indicator in fine_indicators):
+                # Re-check with fine keywords
+                for kw in ["mức phạt", "vi phạm", "đèn đỏ", "vượt đèn", "muc phat", "vuot den", "phat", "vuot", "den", "muc"]:
+                    if self._keyword_in(query_lower, query_ascii, kw):
+                        return ("search_fine", 0.9)
+            # Only return greeting if query is very short (<= 3 words)
+            if len(query_words) > 3:
+                # If long query classified as greeting, it's probably wrong - use general
+                return ("general_query", 0.5)
+        
+        # Ưu tiên intent pháp luật nếu có nhiều tín hiệu liên quan DB
+        query_lower_for_boost = query.lower()
+        if "search_legal" in {model_intent, keyword_intent}:
+            if any(kw in query_lower_for_boost for kw in LEGAL_KEYWORDS):
+                chosen_intent = "search_legal"
+                confidence = max(confidence, model_confidence, keyword_confidence, 0.8)
+        
+        # Confidence calibration: adjust confidence based on query characteristics
+        confidence = self._calibrate_confidence(chosen_intent, confidence, query)
+        
+        return (chosen_intent, confidence)
+    
+    def _calibrate_confidence(self, intent: str, base_confidence: float, query: str) -> float:
+        """
+        Calibrate confidence score based on query characteristics.
+        
+        Args:
+            intent: Classified intent.
+            base_confidence: Base confidence score.
+            query: Original query.
+        
+        Returns:
+            Calibrated confidence score.
+        """
+        confidence = base_confidence
+        query_lower = query.lower()
+        query_words = query.split()
+        
+        # Boost confidence for longer, more specific queries
+        if len(query_words) >= 5:
+            confidence = min(1.0, confidence + 0.05)
+        
+        # Reduce confidence for very short queries
+        if len(query_words) <= 2:
+            confidence = max(0.3, confidence - 0.1)
+        
+        # Boost confidence if query contains intent-specific keywords
+        intent_keywords = {
+            "search_fine": ["phạt", "mức phạt", "vi phạm"],
+            "search_procedure": ["thủ tục", "hồ sơ", "giấy tờ"],
+            "search_office": ["địa chỉ", "công an", "đơn vị"],
+            "search_advisory": ["cảnh báo", "lừa đảo", "scam"],
+            "search_legal": LEGAL_KEYWORDS,
+        }
+        
+        if intent in intent_keywords:
+            keywords = intent_keywords[intent]
+            if any(kw in query_lower for kw in keywords):
+                confidence = min(1.0, confidence + 0.05)
+        
+        return confidence
+    
+    def _personalize_query(
+        self,
+        query: str,
+        intent: str,
+        context: Optional[Dict[str, Any]],
+        session_id: Optional[str]
+    ) -> str:
+        """
+        Personalize query based on user history and session context.
+        
+        Args:
+            query: Original query.
+            intent: Detected intent.
+            context: Conversation context.
+            session_id: Session ID.
+        
+        Returns:
+            Personalized query.
+        """
+        if not context or not session_id:
+            return query
+        
+        # Get user preferences from context
+        entities = context.get("entities", {})
+        intents = context.get("intents", [])
+        
+        # If user frequently asks about same intent, boost related terms
+        if intents and len(intents) >= 2:
+            most_common_intent = max(set(intents), key=intents.count)
+            if most_common_intent == intent:
+                # User prefers this intent - query is already personalized
+                pass
+        
+        # Add context entities to query if missing
+        enhanced_parts = [query]
+        
+        if intent == "search_fine" and "fine_code" in entities:
+            fine_code = entities["fine_code"]
+            if fine_code not in query.lower():
+                enhanced_parts.append(fine_code)
+        
+        return " ".join(enhanced_parts)
+    
+    def _model_based_intent(self, query: str) -> Tuple[Optional[str], float]:
+        """Phân loại ý định bằng model đã huấn luyện nếu có."""
+        if not self.intent_classifier:
+            return (None, 0.0)
+        try:
+            predicted_proba = getattr(self.intent_classifier, "predict_proba", None)
+            if not predicted_proba:
+                return (None, 0.0)
+            probs = predicted_proba([query])[0]
+            classes = self._intent_classes()
+            if not classes:
+                return (None, 0.0)
+            max_idx = int(np.argmax(probs))
+            return (classes[max_idx], float(probs[max_idx]))
+        except Exception:
+            return (None, 0.0)
+
+    def _intent_classes(self) -> Optional[List[str]]:
+        if not self.intent_classifier:
+            return None
+        if hasattr(self.intent_classifier, "classes_"):
+            return list(self.intent_classifier.classes_)
+        named_steps = getattr(self.intent_classifier, "named_steps", {})
+        clf = named_steps.get("clf") if isinstance(named_steps, dict) else None
+        if clf and hasattr(clf, "classes_"):
+            return list(clf.classes_)
+        return None
+    
+    def _keyword_based_intent(self, query: str) -> Tuple[str, float]:
+        """Fallback keyword-based intent classification."""
+        # Use original query (lowercase) to preserve Vietnamese characters
+        query_lower = query.lower().strip()
+        query_ascii = self._remove_accents(query_lower)
+        query_words = query_lower.split()
+        
+        # Check for keywords - prioritize fine-related queries FIRST
+        # Check on original query to preserve Vietnamese characters
+        # Check longer phrases first, then single words
+        fine_keywords = ["mức phạt", "vi phạm", "đèn đỏ", "nồng độ cồn", "mũ bảo hiểm", "tốc độ", "bằng lái", "vượt đèn", "mức phạt vượt"]
+        fine_keywords_ascii = [self._remove_accents(kw) for kw in fine_keywords]
+        fine_single_words = ["phạt", "vượt", "đèn", "mức", "phat", "vuot", "den"]
+        
+        # Check multi-word keywords first
+        has_fine_keywords = False
+        for kw, kw_ascii in zip(fine_keywords, fine_keywords_ascii):
+            if self._keyword_in(query_lower, query_ascii, kw) or kw_ascii in query_ascii:
+                return ("search_fine", 0.95)  # Very high confidence
+        # Then check single words - check ALL of them, not just first match
+        for kw in fine_single_words:
+            if self._keyword_in(query_lower, query_ascii, kw):
+                has_fine_keywords = True
+                # Return immediately if found
+                return ("search_fine", 0.9)
+        
+        has_procedure_keywords = any(
+            self._keyword_in(query_lower, query_ascii, kw) for kw in
+            ["thủ tục", "hồ sơ", "điều kiện", "cư trú", "antt", "pccc", "thu tuc", "ho so", "dieu kien", "cu tru"]
+        )
+        if has_procedure_keywords:
+            return ("search_procedure", 0.8)
+        
+        # Check advisory keywords first to avoid conflict with "công an" in office keywords
+        has_advisory_keywords = any(
+            self._keyword_in(query_lower, query_ascii, kw) for kw in
+            ["cảnh báo", "lừa đảo", "scam", "mạo danh", "thủ đoạn", "cảnh giác", "canh bao", "lua dao", "mao danh", "thu doan", "canh giac"]
+        )
+        if has_advisory_keywords:
+            return ("search_advisory", 0.8)
+        
+        has_office_keywords = any(
+            self._keyword_in(query_lower, query_ascii, kw) for kw in
+            ["địa chỉ", "điểm tiếp dân", "công an", "số điện thoại", "giờ làm việc", "dia chi", "diem tiep dan", "cong an", "so dien thoai", "gio lam viec"]
+        )
+        if has_office_keywords:
+            return ("search_office", 0.8)
+        
+        # Check legal keywords (check BEFORE advisory to avoid "công an" conflict)
+        has_legal_keywords = any(
+            self._keyword_in(query_lower, query_ascii, kw) for kw in
+            ["quyết định", "quy định", "thông tư", "nghị quyết", "văn bản pháp luật", "văn bản quy phạm", "điều lệnh",
+             "kỷ luật đảng viên", "kỷ luật", "xử lý kỷ luật", "hình thức kỷ luật", "mức kỷ luật",
+             "quyết định 69", "quyết định 264", "qd 69", "qd 264", "thông tư 02", "tt 02",
+             "quy định kỷ luật", "kỷ luật đảng", "k��� luật cán bộ", "xử lý vi phạm",
+             "quyet dinh", "quy dinh", "thong tu", "nghi quyet", "van ban phap luat", "van ban quy pham", "dieu lenh",
+             "ky luat dang vien", "ky luat", "xu ly ky luat", "hinh thuc ky luat", "muc ky luat",
+             "quyet dinh 69", "quyet dinh 264", "qd 69", "qd 264", "thong tu 02", "tt 02",
+             "quy dinh ky luat", "ky luat dang", "ky luat can bo", "xu ly vi pham"]
+        )
+        if has_legal_keywords:
+            return ("search_legal", 0.85)
+        
+        # Only treat as greeting if it's VERY short (<= 3 words) and ONLY contains greeting words
+        # AND does NOT contain any other keywords
+        has_any_keyword = (has_fine_keywords or has_procedure_keywords or 
+                          has_office_keywords or has_advisory_keywords or has_legal_keywords)
+        
+        if (len(query_words) <= 3 and 
+            any(self._keyword_in(query_lower, query_ascii, kw) for kw in ["xin chào", "chào", "hello", "hi", "xin chao", "chao"]) and
+            not has_any_keyword):
+            return ("greeting", 0.9)
+        
+        return ("general_query", 0.5)
+    
+    def extract_keywords(self, query: str) -> List[str]:
+        """Extract keywords from query for search."""
+        # Remove common stopwords
+        stopwords = {"là", "gì", "bao nhiêu", "như thế nào", "ở đâu", "của", "và", "hoặc", "tôi", "bạn"}
+        
+        words = re.findall(r'\b\w+\b', query.lower())
+        keywords = [w for w in words if w not in stopwords and len(w) > 2]
+        
+        return keywords
+    
+    def search_by_intent(self, intent: str, query: str, limit: int = 5) -> Dict[str, Any]:
+        """Search based on classified intent."""
+        # Use original query for better matching, especially for Vietnamese text
+        keywords = query.strip()
+        # Also try with extracted keywords as fallback
+        extracted = " ".join(self.extract_keywords(query))
+        if extracted and len(extracted) > 2:
+            keywords = f"{keywords} {extracted}"
+        
+        results = []
+        
+        if intent == "search_fine":
+            qs = Fine.objects.all()
+            text_fields = ["name", "code", "article", "decree", "remedial"]
+            search_results = search_with_ml(qs, keywords, text_fields, top_k=limit, min_score=0.1)
+            results = [{"type": "fine", "data": {
+                "id": f.id,
+                "name": f.name,
+                "code": f.code,
+                "min_fine": float(f.min_fine) if f.min_fine else None,
+                "max_fine": float(f.max_fine) if f.max_fine else None,
+                "fine_amount_formatted": format_fine_amount(
+                    float(f.min_fine) if f.min_fine else None,
+                    float(f.max_fine) if f.max_fine else None
+                ),
+                "article": f.article,
+                "decree": f.decree,
+            }} for f in search_results]
+        
+        elif intent == "search_procedure":
+            qs = Procedure.objects.all()
+            text_fields = ["title", "domain", "conditions", "dossier"]
+            search_results = search_with_ml(qs, keywords, text_fields, top_k=limit, min_score=0.1)
+            results = [{"type": "procedure", "data": {
+                "id": p.id,
+                "title": p.title,
+                "domain": p.domain,
+                "level": p.level,
+            }} for p in search_results]
+        
+        elif intent == "search_office":
+            qs = Office.objects.all()
+            text_fields = ["unit_name", "address", "district", "service_scope"]
+            search_results = search_with_ml(qs, keywords, text_fields, top_k=limit, min_score=0.1)
+            results = [{"type": "office", "data": {
+                "id": o.id,
+                "unit_name": o.unit_name,
+                "address": o.address,
+                "district": o.district,
+                "phone": o.phone,
+                "working_hours": o.working_hours,
+            }} for o in search_results]
+        
+        elif intent == "search_advisory":
+            qs = Advisory.objects.all()
+            text_fields = ["title", "summary"]
+            search_results = search_with_ml(qs, keywords, text_fields, top_k=limit, min_score=0.1)
+            results = [{"type": "advisory", "data": {
+                "id": a.id,
+                "title": a.title,
+                "summary": a.summary,
+            }} for a in search_results]
+        
+        return {
+            "intent": intent,
+            "query": query,
+            "keywords": keywords,
+            "results": results,
+            "count": len(results)
+        }
+    
+    def _serialize_document(self, doc: Any, content_type: str) -> Dict[str, Any]:
+        """Convert Django model instance to JSON-serializable dict."""
+        base = {"id": getattr(doc, "id", None)}
+        content_type = (content_type or "").lower()
+        
+        def _to_iso(value):
+            if value is None:
+                return None
+            if hasattr(value, "isoformat"):
+                return value.isoformat()
+            return value
+        
+        if content_type == "procedure":
+            base.update({
+                "title": getattr(doc, "title", ""),
+                "domain": getattr(doc, "domain", ""),
+                "level": getattr(doc, "level", ""),
+                "conditions": getattr(doc, "conditions", ""),
+                "dossier": getattr(doc, "dossier", ""),
+                "fee": getattr(doc, "fee", ""),
+                "duration": getattr(doc, "duration", ""),
+                "authority": getattr(doc, "authority", ""),
+                "source_url": getattr(doc, "source_url", ""),
+            })
+        elif content_type == "fine":
+            min_fine = float(doc.min_fine) if getattr(doc, "min_fine", None) is not None else None
+            max_fine = float(doc.max_fine) if getattr(doc, "max_fine", None) is not None else None
+            base.update({
+                "name": getattr(doc, "name", ""),
+                "code": getattr(doc, "code", ""),
+                "min_fine": min_fine,
+                "max_fine": max_fine,
+                "fine_amount_formatted": format_fine_amount(min_fine, max_fine),
+                "article": getattr(doc, "article", ""),
+                "decree": getattr(doc, "decree", ""),
+            })
+        elif content_type == "office":
+            base.update({
+                "unit_name": getattr(doc, "unit_name", ""),
+                "address": getattr(doc, "address", ""),
+                "district": getattr(doc, "district", ""),
+                "phone": getattr(doc, "phone", ""),
+                "working_hours": getattr(doc, "working_hours", ""),
+            })
+        elif content_type == "advisory":
+            base.update({
+                "title": getattr(doc, "title", ""),
+                "summary": getattr(doc, "summary", ""),
+                "source_url": getattr(doc, "source_url", ""),
+                "published_at": _to_iso(getattr(doc, "published_at", None)),
+            })
+        else:
+            # Fallback: include common attributes but skip non-serializable ones
+            safe_attrs = [
+                "title", "summary", "description", "domain", "level", "conditions",
+                "dossier", "fee", "duration", "authority", "unit_name", "address",
+                "district", "phone", "working_hours", "source_url", "published_at"
+            ]
+            for attr in safe_attrs:
+                if hasattr(doc, attr):
+                    value = getattr(doc, attr)
+                    if attr == "published_at":
+                        base[attr] = _to_iso(value)
+                    elif isinstance(value, (str, int, float, type(None))):
+                        base[attr] = value
+        
+        return base
+    
+    def generate_response(self, query: str, session_id: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Generate chatbot response for user query with context awareness.
+        
+        Args:
+            query: User query string.
+            session_id: Optional session ID for context management.
+        
+        Returns:
+            Dict with message, intent, results, and session_id.
+        """
+        query = query.strip()
+        original_query = query
+        
+        # Get context if session_id provided
+        context_messages = []
+        context_dict = None
+        if session_id:
+            try:
+                from hue_portal.chatbot.context_manager import ConversationContext
+                from hue_portal.chatbot.entity_extraction import resolve_pronouns, extract_all_entities
+                
+                # Get recent messages
+                recent_messages = ConversationContext.get_recent_messages(session_id, limit=10)
+                context_messages = [
+                    {
+                        "role": msg.role,
+                        "content": msg.content,
+                        "intent": msg.intent,
+                        "entities": msg.entities
+                    }
+                    for msg in recent_messages
+                ]
+                
+                # Build context dictionary for intent classification
+                context_dict = ConversationContext.get_context_summary(session_id, max_messages=5)
+                
+                # Resolve pronouns in query
+                if context_messages:
+                    query = resolve_pronouns(query, context_messages)
+                    if query != original_query:
+                        print(f"Query enhanced with context: '{original_query}' -> '{query}'")
+            except Exception as e:
+                print(f"Error loading context: {e}")
+        
+        # Classify intent FIRST (use enhanced query and context)
+        intent, confidence = self.classify_intent(query, context=context_dict)
+        
+        # Personalize response based on context and user history
+        personalized_query = self._personalize_query(query, intent, context_dict, session_id)
+        if personalized_query != query:
+            query = personalized_query
+        
+        # Only handle greetings if it's REALLY a simple greeting (very short, no other keywords)
+        query_lower = query.lower().strip()
+        query_words = query_lower.split()
+        
+        # Check if it contains keywords that indicate it's NOT a greeting
+        has_fine_keywords = any(kw in query_lower for kw in ["phạt", "mức phạt", "vi phạm", "đèn đỏ", "nồng độ cồn", "mũ bảo hiểm", "tốc độ", "vượt"])
+        has_procedure_keywords = any(kw in query_lower for kw in ["thủ tục", "hồ sơ", "điều kiện", "cư trú", "antt", "pccc"])
+        # Check advisory keywords first to avoid conflict with "công an" in office keywords
+        has_advisory_keywords = any(kw in query_lower for kw in ["cảnh báo", "lừa đảo", "scam", "mạo danh", "thủ đoạn", "cảnh giác"])
+        has_office_keywords = any(kw in query_lower for kw in ["địa chỉ", "công an", "số điện thoại", "giờ làm việc"])
+        has_legal_keywords = any(kw in query_lower for kw in ["quyết định", "quy định", "thông tư", "kỷ luật đảng viên", "kỷ luật", "qd 69", "qd 264", "thông tư 02", "điều lệnh", "văn bản pháp luật"])
+        
+        # Only treat as greeting if it's very short AND has no other keywords AND classified as greeting
+        is_simple_greeting = (len(query_words) <= 3 and 
+                             any(greeting in query_lower for greeting in ["xin chào", "chào", "hello", "hi"]) and
+                             not (has_fine_keywords or has_procedure_keywords or has_office_keywords or has_advisory_keywords or has_legal_keywords))
+        
+        if is_simple_greeting and intent == "greeting":
+            response = {
+                "message": RESPONSE_TEMPLATES["greeting"],
+                "intent": "greeting",
+                "results": [],
+                "count": 0,
+                "session_id": session_id
+            }
+            
+            # Save conversation if session_id provided
+            if session_id:
+                try:
+                    from hue_portal.chatbot.context_manager import ConversationContext
+                    from hue_portal.chatbot.entity_extraction import extract_all_entities
+                    
+                    # Save user message
+                    entities = extract_all_entities(original_query)
+                    ConversationContext.add_message(
+                        session_id=session_id,
+                        role="user",
+                        content=original_query,
+                        intent=intent,
+                        entities=entities
+                    )
+                    
+                    # Save bot response
+                    ConversationContext.add_message(
+                        session_id=session_id,
+                        role="bot",
+                        content=RESPONSE_TEMPLATES["greeting"],
+                        intent=intent
+                    )
+                except Exception as e:
+                    print(f"Error saving conversation: {e}")
+            
+            return response
+        
+        # Try RAG pipeline first (if embeddings available)
+        use_rag = True
+        try:
+            from hue_portal.core.rag import rag_pipeline
+            # Build context list for RAG
+            rag_context = None
+            if context_messages:
+                rag_context = context_messages
+            rag_result = rag_pipeline(query, intent, top_k=5, min_confidence=confidence, context=rag_context, use_llm=True)
+            
+            # Use RAG answer if available (even with count=0 for general conversation)
+            if rag_result.get("answer") and (rag_result["count"] > 0 or rag_result.get("answer", "").strip()):
+                # Use RAG-generated answer
+                documents = rag_result["documents"][:5]
+                results = [
+                    {
+                        "type": rag_result["content_type"],
+                        "data": self._serialize_document(doc, rag_result["content_type"])
+                    } for doc in documents
+                ]
+                
+                # Add best_match flag and relevance_scores
+                best_match_index = 0 if documents else None
+                relevance_scores = []
+                for i, doc in enumerate(documents):
+                    score = getattr(doc, "_hybrid_score", getattr(doc, "_ml_score", 0.0))
+                    relevance_scores.append({
+                        "index": i,
+                        "score": float(score) if score else 0.0,
+                        "is_best_match": i == 0
+                    })
+                
+                response = {
+                    "message": rag_result["answer"],
+                    "intent": intent,
+                    "confidence": rag_result["confidence"],
+                    "results": results,
+                    "count": rag_result["count"],
+                    "best_match": best_match_index,
+                    "relevance_scores": relevance_scores,
+                    "session_id": session_id
+                }
+                
+                # Save conversation if session_id provided
+                if session_id:
+                    try:
+                        from hue_portal.chatbot.context_manager import ConversationContext
+                        from hue_portal.chatbot.entity_extraction import extract_all_entities
+                        
+                        # Save user message
+                        entities = extract_all_entities(original_query)
+                        ConversationContext.add_message(
+                            session_id=session_id,
+                            role="user",
+                            content=original_query,
+                            intent=intent,
+                            entities=entities
+                        )
+                        
+                        # Save bot response
+                        ConversationContext.add_message(
+                            session_id=session_id,
+                            role="bot",
+                            content=rag_result["answer"],
+                            intent=intent
+                        )
+                    except Exception as e:
+                        print(f"Error saving conversation: {e}")
+                
+                return response
+        except Exception as e:
+            # Fallback to original search if RAG fails
+            print(f"RAG pipeline not available, using original search: {e}")
+            use_rag = False
+        
+        # Search based on intent (original method)
+        search_result = self.search_by_intent(intent, query, limit=5)
+        
+        # Generate response message
+        if search_result["count"] > 0:
+            template = RESPONSE_TEMPLATES.get(intent, RESPONSE_TEMPLATES["general_query"])
+            message = template.format(
+                count=search_result["count"],
+                query=query
+            )
+        else:
+            message = RESPONSE_TEMPLATES["no_results"].format(query=query)
+        
+        # Add best_match flag and relevance_scores for search results
+        best_match_index = 0 if search_result["count"] > 0 else None
+        relevance_scores = []
+        for i, result in enumerate(search_result["results"][:5]):
+            # Try to get score from result data if available
+            score = 0.0
+            if isinstance(result, dict) and "data" in result:
+                # Score might be in the data or we can infer from position
+                score = 1.0 - (i * 0.1)  # Decreasing score for lower positions
+            relevance_scores.append({
+                "index": i,
+                "score": score,
+                "is_best_match": i == 0
+            })
+        
+        response = {
+            "message": message,
+            "intent": intent,
+            "confidence": confidence,
+            "results": search_result["results"],
+            "count": search_result["count"],
+            "best_match": best_match_index,
+            "relevance_scores": relevance_scores,
+            "session_id": session_id
+        }
+        
+        # Save conversation if session_id provided
+        if session_id:
+            try:
+                from hue_portal.chatbot.context_manager import ConversationContext
+                from hue_portal.chatbot.entity_extraction import extract_all_entities
+                
+                # Save user message
+                entities = extract_all_entities(original_query)
+                ConversationContext.add_message(
+                    session_id=session_id,
+                    role="user",
+                    content=original_query,
+                    intent=intent,
+                    entities=entities
+                )
+                
+                # Save bot response
+                ConversationContext.add_message(
+                    session_id=session_id,
+                    role="bot",
+                    content=message,
+                    intent=intent
+                )
+            except Exception as e:
+                print(f"Error saving conversation: {e}")
+        
+        return response
+
+
+# Global chatbot instance
+_chatbot_instance = None
+
+def get_chatbot() -> Chatbot:
+    """Get or create chatbot instance."""
+    global _chatbot_instance
+    if _chatbot_instance is None:
+        _chatbot_instance = Chatbot()
+    return _chatbot_instance